pezkuwichain/pezkuwi-sdk
1
Fork 0
mirror of https://gitlab.com/pezkuwichain/pezkuwi-sdk.git synced 2026-10-20 18:56:23 +00:00
Code Releases Activity

usdt-bridge: never drop a withdrawal record on a transient RPC failure

Browse Source 
process_withdrawal_block's reserve/circuit-breaker check used to `continue`
past a Polkadot connect failure without ever calling db.add_withdrawal() -
a transient RPC error (confirmed live: a public-endpoint rate limit) meant
an already-finalized Pezkuwi withdrawal was silently untracked, with no
pending_multisig_approval entry for the signatories to ever see. It now
always records the withdrawal; an unreachable reserve check is treated as
"unverified" (never auto-paid, flagged distinctly from a confirmed
shortfall) rather than as a reason to drop the record.

Also add bounded exponential backoff (2s..32s, 6 attempts) to
connect_to_chain instead of failing on the first error - every caller
either bubbles the failure through a listener's outer reconnect loop or
out of main() to a systemd restart, so a single blip was tearing down and
fully re-initializing both listeners instead of just waiting it out.

Finally, support POLKADOT_RPC_URL/PEZKUWI_RPC_URL env var overrides for
the configured RPC endpoints, so a dedicated/API-keyed provider URL (e.g.
Dwellir, adopted after the public Polkadot Asset Hub endpoint started
rate-limiting us) can live in a tightly-permissioned EnvironmentFile
instead of the git-tracked bridge_config.json - same pattern already used
for PEZBRIDGE_TELEGRAM_TOKEN and AUTOMATION_KEY_MNEMONIC.
This commit is contained in:
Satoshi Qazi Muhammed committed 2026-07-16 08:51:23 -07:00
1 parent 8265794150
commit 1771162793
1 file changed
+83 -25
Download Patch File Download Diff File Expand all files Collapse all files
tools/usdt-bridge/src/main.rs
+83 -25
View File
@@ -529,12 +529,38 @@ fn init_db(db_path: &PathBuf) -> Result<()> {
// Chain Operations
// ============================================================================
/// Retries with exponential backoff (2s, 4s, 8s, 16s, 32s; capped, 6 attempts total) rather than
/// failing on the first transient error. This matters because every caller here either bubbles a
/// failure straight up through a `?` to the listener's outer reconnect loop (see listen_deposits/
/// listen_withdrawals) or all the way out of main() to a systemd restart - a single blip (a rate
/// limit, a DNS hiccup) would otherwise tear down and fully re-initialize both listeners instead
/// of just waiting the blip out. Observed live 2026-07-16: the public Polkadot Asset Hub RPC
/// briefly 429'd, which without this retry crashed the whole process and then kept re-crashing
/// every RestartSec, hammering the same rate-limited endpoint harder instead of backing off.
async fn connect_to_chain(url: &str) -> Result<OnlineClient<SubstrateConfig>> {
info!("Connecting to {}...", url);
let client = OnlineClient::<SubstrateConfig>::from_url(url).await
.context(format!("Failed to connect to {}", url))?;
info!("Connected successfully");
Ok(client)
const MAX_ATTEMPTS: u32 = 6;
let mut delay = Duration::from_secs(2);
for attempt in 1..=MAX_ATTEMPTS {
info!("Connecting to {}... (attempt {attempt}/{MAX_ATTEMPTS})", url);
match OnlineClient::<SubstrateConfig>::from_url(url).await {
Ok(client) => {
info!("Connected successfully");
return Ok(client);
}
Err(e) if attempt < MAX_ATTEMPTS => {
warn!("Connect attempt {attempt}/{MAX_ATTEMPTS} to {url} failed: {e} - retrying in {}s", delay.as_secs());
tokio::time::sleep(delay).await;
delay = (delay * 2).min(Duration::from_secs(60));
}
Err(e) => {
return Err(e).context(format!("Failed to connect to {url} after {MAX_ATTEMPTS} attempts"));
}
}
}
unreachable!("loop always returns via Ok or the final Err arm")
}
/// Get asset balance using dynamic API
@@ -1433,21 +1459,36 @@ async fn process_withdrawal_block(
// wallet actually holds. If this ever trips, the peg is already
// under-collateralized (e.g. wUSDT minted without a matching real
// deposit) - that must halt and page someone, not degrade silently.
let polkadot_client = match connect_to_chain(&config.polkadot_rpc).await {
Ok(c) => c,
Err(e) => {
error!(" ❌ Failed to connect to Polkadot: {}", e);
continue;
}
//
// Note this reserve check is best-effort only: the withdrawal itself
// is already finalized on Pezkuwi by the time we get here, so a
// transient RPC hiccup while trying to verify the Polkadot-side
// reserve must NEVER `continue` past recording it - that would drop
// a real, already-committed withdrawal on the floor with no
// pending_multisig_approval entry anywhere (exactly the silent-loss
// failure mode fixed for the subscription-stall bug). The signatories
// verify the real reserve themselves in pwap-web before approving, so
// an unverified reserve here just means "never auto-pay, always
// human-review" - it does not block recording the request.
let polkadot_client = connect_to_chain(&config.polkadot_rpc).await.ok();
let reserve_known = polkadot_client.is_some();
let reserve = match &polkadot_client {
Some(client) => get_asset_balance(
client,
config.polkadot_usdt_asset_id,
polkadot_bridge_addr,
).await.unwrap_or(0),
None => 0,
};
let reserve = get_asset_balance(
&polkadot_client,
config.polkadot_usdt_asset_id,
polkadot_bridge_addr,
).await.unwrap_or(0);
if reserve < net_amount {
if !reserve_known {
error!(" ❌ Failed to connect to Polkadot to verify reserve - recording withdrawal as pending_multisig_approval anyway (reserve NOT confirmed, signatories must check manually)");
notify_telegram(config, &format!(
"⚠️ Could not verify Polkadot reserve for a {} USDT withdrawal from {} (RPC connect failed). Recorded as pending_multisig_approval - reserve NOT confirmed, signatories must check manually before approving.",
amount as f64 / 1_000_000.0, from_str
)).await;
} else if reserve < net_amount {
let shortfall_msg = format!(
"🚨 RESERVE SHORTFALL: custody holds {} USDT, need {} USDT to honor a pending {} USDT withdrawal from {}. \
This must be reviewed by the signatories before any approval - do NOT approve as-is.",
@@ -1462,9 +1503,9 @@ async fn process_withdrawal_block(
match convert_ss58_prefix(&from_str, 0) {
Ok(polkadot_recipient) => {
let now = Utc::now();
// Never attempt auto-pay against an already-known reserve shortfall -
// that's exactly the case that must always go to human review.
let auto_pay_attempt = if reserve < net_amount {
// Never attempt auto-pay against an unverified or known-short reserve -
// both cases must always go to human review.
let auto_pay_attempt = if !reserve_known || reserve < net_amount {
None
} else {
match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) {
@@ -1560,8 +1601,12 @@ async fn process_withdrawal_block(
" 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot",
amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount
);
info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
reserve as f64 / 1_000_000.0);
if reserve_known {
info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
reserve as f64 / 1_000_000.0);
} else {
info!(" Reserve could NOT be verified (Polkadot RPC unreachable at detection time) - signatories must check manually via pwap-web.");
}
db.add_withdrawal(WithdrawalRecord {
id: db.next_withdrawal_id(),
@@ -1573,7 +1618,9 @@ async fn process_withdrawal_block(
fee,
net_amount,
polkadot_tx_hash: None,
status: if reserve < net_amount {
status: if !reserve_known {
"pending_multisig_approval_reserve_unverified".to_string()
} else if reserve < net_amount {
"pending_multisig_approval_reserve_shortfall".to_string()
} else {
"pending_multisig_approval".to_string()
@@ -1822,7 +1869,7 @@ async fn main() -> Result<()> {
let cli = Cli::parse();
// Load or create config
let config: BridgeConfig = if cli.config.exists() {
let mut config: BridgeConfig = if cli.config.exists() {
let content = std::fs::read_to_string(&cli.config)?;
serde_json::from_str(&content)?
} else {
@@ -1833,6 +1880,17 @@ async fn main() -> Result<()> {
default_config
};
// Dedicated/API-keyed RPC endpoints (e.g. a Dwellir URL) carry a credential in the URL itself,
// so - same reasoning as PEZBRIDGE_TELEGRAM_TOKEN/AUTOMATION_KEY_MNEMONIC below - they belong
// in an env var backed by a tightly-permissioned EnvironmentFile, not in the git-tracked JSON
// config. Unset means "use whatever bridge_config.json says" (the public endpoint by default).
if let Ok(url) = std::env::var("POLKADOT_RPC_URL") {
config.polkadot_rpc = url;
}
if let Ok(url) = std::env::var("PEZKUWI_RPC_URL") {
config.pezkuwi_rpc = url;
}
match cli.command {
Commands::GenerateWallet { output } => {
generate_wallet(&output)?;