mirror of
https://gitlab.com/pezkuwichain/pezkuwi-sdk.git
synced 2026-10-20 18:56:23 +00:00
usdt-bridge: record over-cap swaps, throttle backfill, gate routine Telegram noise
Same silent-loss pattern as the previous connect-failure fix, found while auditing further: amount > max_single_tx (both deposit and withdrawal) warned and Telegram-alerted but never called db.add_deposit/add_withdrawal - "not auto-recorded" was literally in the log line. Now always recorded (pending_multisig_approval_amount_exceeds_cap), same as every other manual-review path. Backfill throttle: catching up many blocks after real downtime fires a tight burst of RPC calls that can trip a metered/shared endpoint's rate limit hard enough to poison the whole connection - observed live. A 150ms pause between backfilled blocks avoids it. Telegram notification threshold: at real user volume, a message for every small successful auto-pay would bury the ones that need a human (failures, over-cap, reserve problems, anything falling to manual 3-of-5) under noise. Added notify_threshold (default 500 USDT) gating ONLY the routine "auto-paid successfully" message on both legs - everything else keeps notifying unconditionally regardless of amount. Also found and fixed a real asymmetry: the withdrawal leg's "awaiting multisig approval" case never notified Telegram at all (the deposit leg always did) - a withdrawal needing manual review must always page someone.
This commit is contained in:
Satoshi Qazi Muhammed committed
1 parent
1771162793
commit
82543bdfab
2 files changed
+82
-19
No files matched your search
Generated
+1
-1
@@ -3278,7 +3278,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -135,12 +135,23 @@ pub struct BridgeConfig {
|
||||
/// Same, for the ops group chat.
|
||||
#[serde(default)]
|
||||
pub ops_chat_id: i64,
|
||||
/// Below this, a SUCCESSFUL auto-pay stays silent on Telegram - at real user volume, a
|
||||
/// message per small routine swap would bury the notifications that actually need a human's
|
||||
/// attention (failures, amounts over max_single_tx, reserve problems, anything that fell to
|
||||
/// manual 3-of-5 review) under noise. Those all notify unconditionally regardless of amount -
|
||||
/// only the routine "everything worked, nothing to look at" case is gated by this.
|
||||
#[serde(default = "default_notify_threshold")]
|
||||
pub notify_threshold: u128,
|
||||
}
|
||||
|
||||
fn default_max_single_tx() -> u128 {
|
||||
50_000_000_000 // 50,000 USDT (6 decimals) - small/mid-size swaps clear without waiting on 3-of-5
|
||||
}
|
||||
|
||||
fn default_notify_threshold() -> u128 {
|
||||
500_000_000 // 500 USDT (6 decimals)
|
||||
}
|
||||
|
||||
fn default_multisig_polkadot() -> String {
|
||||
"15sF76THfpefUaKomHZSpssayRbsp6Yt6ESgMrLjzJCmpe66".to_string()
|
||||
}
|
||||
@@ -181,6 +192,7 @@ impl Default for BridgeConfig {
|
||||
auto_pay_daily_cap: default_auto_pay_daily_cap(),
|
||||
circuit_breaker_window_minutes: default_circuit_breaker_window_minutes(),
|
||||
circuit_breaker_fraction: default_circuit_breaker_fraction(),
|
||||
notify_threshold: default_notify_threshold(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1085,14 +1097,30 @@ async fn process_deposit_block(
|
||||
warn!(" ⚠️ Already processed ({}), skipping - refusing to mint twice", tx_hash);
|
||||
} else if amount > config.max_single_tx {
|
||||
warn!(
|
||||
" ⚠️ Amount {} exceeds max_single_tx cap {} - leaving for manual multi-party review, NOT auto-processing",
|
||||
" ⚠️ Amount {} exceeds max_single_tx cap {} - recording as pending_multisig_approval_amount_exceeds_cap, needs manual multi-party review",
|
||||
amount as f64 / 1_000_000.0,
|
||||
config.max_single_tx as f64 / 1_000_000.0
|
||||
);
|
||||
notify_telegram(config, &format!(
|
||||
"📥 Deposit {} USDT from {} exceeds max_single_tx ({} USDT) - needs manual multi-party review, not auto-recorded.",
|
||||
"📥 Deposit {} USDT from {} exceeds max_single_tx ({} USDT) - recorded, needs manual multi-party review.",
|
||||
amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0
|
||||
)).await;
|
||||
let fee = calculate_fee(amount, config.fee_basis_points);
|
||||
let net_amount = amount - fee;
|
||||
db.add_deposit(DepositRecord {
|
||||
id: db.next_deposit_id(),
|
||||
polkadot_tx_hash: tx_hash.clone(),
|
||||
polkadot_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
pezkuwi_tx_hash: None,
|
||||
status: "pending_multisig_approval_amount_exceeds_cap".to_string(),
|
||||
created_at: Utc::now().to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
} else if amount >= config.min_deposit {
|
||||
let fee = calculate_fee(amount, config.fee_basis_points);
|
||||
let net_amount = amount - fee;
|
||||
@@ -1158,7 +1186,7 @@ async fn process_deposit_block(
|
||||
db.save(&config.db_path)?;
|
||||
if let Some(reason) = breaker_tripped {
|
||||
notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (deposit/wUSDT leg): {reason}. All further deposits now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await;
|
||||
} else {
|
||||
} else if amount >= config.notify_threshold {
|
||||
notify_telegram(config, &format!(
|
||||
"✅ Auto-paid deposit: {} USDT from {} -> {} wUSDT (net of fee) to {}. tx {}",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash
|
||||
@@ -1303,6 +1331,13 @@ async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext)
|
||||
info!("Backfilling Polkadot blocks {} to {head_number} before going live (resuming from checkpoint)", last_processed + 1);
|
||||
for n in (last_processed + 1)..=head_number {
|
||||
backfill_deposit_block(config, automation, &client, &legacy_rpc, n).await?;
|
||||
// A public/shared RPC endpoint (or a metered provider's free tier) can rate-limit
|
||||
// a tight burst of per-block calls hard enough to poison the whole connection
|
||||
// ("background task closed... restart required"), killing this entire listener
|
||||
// over what would otherwise be a routine catch-up after any real downtime -
|
||||
// observed live 2026-07-16. A small per-block pause costs little against a large
|
||||
// gap and avoids that failure mode entirely.
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
}
|
||||
head_number
|
||||
} else {
|
||||
@@ -1358,6 +1393,7 @@ async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext)
|
||||
)).await;
|
||||
for n in (last_seen + 1)..block_number {
|
||||
backfill_deposit_block(config, automation, &client, &legacy_rpc, n).await?;
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1434,19 +1470,6 @@ async fn process_withdrawal_block(
|
||||
continue;
|
||||
}
|
||||
|
||||
if amount > config.max_single_tx {
|
||||
warn!(
|
||||
" ⚠️ Amount {} exceeds max_single_tx cap {} - leaving for manual multi-party review, NOT auto-processing",
|
||||
amount as f64 / 1_000_000.0,
|
||||
config.max_single_tx as f64 / 1_000_000.0
|
||||
);
|
||||
notify_telegram(config, &format!(
|
||||
"📤 Withdrawal {} USDT from {} exceeds max_single_tx ({} USDT) - needs manual multi-party review, not auto-recorded.",
|
||||
amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0
|
||||
)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
if amount < config.min_withdraw {
|
||||
warn!(" ⚠️ Amount below minimum, skipping");
|
||||
continue;
|
||||
@@ -1455,6 +1478,35 @@ async fn process_withdrawal_block(
|
||||
let fee = calculate_fee(amount, config.fee_basis_points);
|
||||
let net_amount = amount - fee;
|
||||
|
||||
if amount > config.max_single_tx {
|
||||
warn!(
|
||||
" ⚠️ Amount {} exceeds max_single_tx cap {} - recording as pending_multisig_approval_amount_exceeds_cap, needs manual multi-party review",
|
||||
amount as f64 / 1_000_000.0,
|
||||
config.max_single_tx as f64 / 1_000_000.0
|
||||
);
|
||||
notify_telegram(config, &format!(
|
||||
"📤 Withdrawal {} USDT from {} exceeds max_single_tx ({} USDT) - recorded, needs manual multi-party review.",
|
||||
amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0
|
||||
)).await;
|
||||
let destination_address = convert_ss58_prefix(&from_str, 0).unwrap_or_default();
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address,
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
polkadot_tx_hash: None,
|
||||
status: "pending_multisig_approval_amount_exceeds_cap".to_string(),
|
||||
created_at: Utc::now().to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Circuit breaker: never release more real USDT than the custody
|
||||
// wallet actually holds. If this ever trips, the peg is already
|
||||
// under-collateralized (e.g. wUSDT minted without a matching real
|
||||
@@ -1568,7 +1620,7 @@ async fn process_withdrawal_block(
|
||||
db.save(&config.db_path)?;
|
||||
if let Some(reason) = breaker_tripped {
|
||||
notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (withdrawal/USDT leg): {reason}. All further withdrawals now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await;
|
||||
} else {
|
||||
} else if amount >= config.notify_threshold {
|
||||
notify_telegram(config, &format!(
|
||||
"✅ Auto-paid withdrawal: {} wUSDT from {} -> {} USDT (net of fee) to {}. tx {}. wUSDT burn still pending - batch via pwap-web when convenient.",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash
|
||||
@@ -1613,7 +1665,7 @@ async fn process_withdrawal_block(
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address: polkadot_recipient,
|
||||
destination_address: polkadot_recipient.clone(),
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
@@ -1629,6 +1681,17 @@ async fn process_withdrawal_block(
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
|
||||
// Unlike the two branches above (reserve_unverified/reserve_shortfall already
|
||||
// notify further up), the plain "automation not configured/eligible" case had no
|
||||
// notification at all until now - a withdrawal needing manual 3-of-5 review must
|
||||
// always page someone, the same as the deposit leg already does.
|
||||
if reserve_known && reserve >= net_amount {
|
||||
notify_telegram(config, &format!(
|
||||
"📤 New withdrawal pending approval: {} USDT from {} -> release {} USDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient
|
||||
)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user