mirror of
https://gitlab.com/pezkuwichain/pezkuwi-sdk.git
synced 2026-10-20 18:56:23 +00:00
usdt-bridge: never drop a withdrawal record on a transient RPC failure
process_withdrawal_block's reserve/circuit-breaker check used to `continue` past a Polkadot connect failure without ever calling db.add_withdrawal() - a transient RPC error (confirmed live: a public-endpoint rate limit) meant an already-finalized Pezkuwi withdrawal was silently untracked, with no pending_multisig_approval entry for the signatories to ever see. It now always records the withdrawal; an unreachable reserve check is treated as "unverified" (never auto-paid, flagged distinctly from a confirmed shortfall) rather than as a reason to drop the record. Also add bounded exponential backoff (2s..32s, 6 attempts) to connect_to_chain instead of failing on the first error - every caller either bubbles the failure through a listener's outer reconnect loop or out of main() to a systemd restart, so a single blip was tearing down and fully re-initializing both listeners instead of just waiting it out. Finally, support POLKADOT_RPC_URL/PEZKUWI_RPC_URL env var overrides for the configured RPC endpoints, so a dedicated/API-keyed provider URL (e.g. Dwellir, adopted after the public Polkadot Asset Hub endpoint started rate-limiting us) can live in a tightly-permissioned EnvironmentFile instead of the git-tracked bridge_config.json - same pattern already used for PEZBRIDGE_TELEGRAM_TOKEN and AUTOMATION_KEY_MNEMONIC.
This commit is contained in:
Satoshi Qazi Muhammed committed
1 parent
8265794150
commit
1771162793
1 file changed
+78
-20
@@ -529,12 +529,38 @@ fn init_db(db_path: &PathBuf) -> Result<()> {
|
||||
// Chain Operations
|
||||
// ============================================================================
|
||||
|
||||
/// Retries with exponential backoff (2s, 4s, 8s, 16s, 32s; capped, 6 attempts total) rather than
|
||||
/// failing on the first transient error. This matters because every caller here either bubbles a
|
||||
/// failure straight up through a `?` to the listener's outer reconnect loop (see listen_deposits/
|
||||
/// listen_withdrawals) or all the way out of main() to a systemd restart - a single blip (a rate
|
||||
/// limit, a DNS hiccup) would otherwise tear down and fully re-initialize both listeners instead
|
||||
/// of just waiting the blip out. Observed live 2026-07-16: the public Polkadot Asset Hub RPC
|
||||
/// briefly 429'd, which without this retry crashed the whole process and then kept re-crashing
|
||||
/// every RestartSec, hammering the same rate-limited endpoint harder instead of backing off.
|
||||
async fn connect_to_chain(url: &str) -> Result<OnlineClient<SubstrateConfig>> {
|
||||
info!("Connecting to {}...", url);
|
||||
let client = OnlineClient::<SubstrateConfig>::from_url(url).await
|
||||
.context(format!("Failed to connect to {}", url))?;
|
||||
const MAX_ATTEMPTS: u32 = 6;
|
||||
let mut delay = Duration::from_secs(2);
|
||||
|
||||
for attempt in 1..=MAX_ATTEMPTS {
|
||||
info!("Connecting to {}... (attempt {attempt}/{MAX_ATTEMPTS})", url);
|
||||
|
||||
match OnlineClient::<SubstrateConfig>::from_url(url).await {
|
||||
Ok(client) => {
|
||||
info!("Connected successfully");
|
||||
Ok(client)
|
||||
return Ok(client);
|
||||
}
|
||||
Err(e) if attempt < MAX_ATTEMPTS => {
|
||||
warn!("Connect attempt {attempt}/{MAX_ATTEMPTS} to {url} failed: {e} - retrying in {}s", delay.as_secs());
|
||||
tokio::time::sleep(delay).await;
|
||||
delay = (delay * 2).min(Duration::from_secs(60));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e).context(format!("Failed to connect to {url} after {MAX_ATTEMPTS} attempts"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unreachable!("loop always returns via Ok or the final Err arm")
|
||||
}
|
||||
|
||||
/// Get asset balance using dynamic API
|
||||
@@ -1433,21 +1459,36 @@ async fn process_withdrawal_block(
|
||||
// wallet actually holds. If this ever trips, the peg is already
|
||||
// under-collateralized (e.g. wUSDT minted without a matching real
|
||||
// deposit) - that must halt and page someone, not degrade silently.
|
||||
let polkadot_client = match connect_to_chain(&config.polkadot_rpc).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!(" ❌ Failed to connect to Polkadot: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
//
|
||||
// Note this reserve check is best-effort only: the withdrawal itself
|
||||
// is already finalized on Pezkuwi by the time we get here, so a
|
||||
// transient RPC hiccup while trying to verify the Polkadot-side
|
||||
// reserve must NEVER `continue` past recording it - that would drop
|
||||
// a real, already-committed withdrawal on the floor with no
|
||||
// pending_multisig_approval entry anywhere (exactly the silent-loss
|
||||
// failure mode fixed for the subscription-stall bug). The signatories
|
||||
// verify the real reserve themselves in pwap-web before approving, so
|
||||
// an unverified reserve here just means "never auto-pay, always
|
||||
// human-review" - it does not block recording the request.
|
||||
let polkadot_client = connect_to_chain(&config.polkadot_rpc).await.ok();
|
||||
let reserve_known = polkadot_client.is_some();
|
||||
|
||||
let reserve = get_asset_balance(
|
||||
&polkadot_client,
|
||||
let reserve = match &polkadot_client {
|
||||
Some(client) => get_asset_balance(
|
||||
client,
|
||||
config.polkadot_usdt_asset_id,
|
||||
polkadot_bridge_addr,
|
||||
).await.unwrap_or(0);
|
||||
).await.unwrap_or(0),
|
||||
None => 0,
|
||||
};
|
||||
|
||||
if reserve < net_amount {
|
||||
if !reserve_known {
|
||||
error!(" ❌ Failed to connect to Polkadot to verify reserve - recording withdrawal as pending_multisig_approval anyway (reserve NOT confirmed, signatories must check manually)");
|
||||
notify_telegram(config, &format!(
|
||||
"⚠️ Could not verify Polkadot reserve for a {} USDT withdrawal from {} (RPC connect failed). Recorded as pending_multisig_approval - reserve NOT confirmed, signatories must check manually before approving.",
|
||||
amount as f64 / 1_000_000.0, from_str
|
||||
)).await;
|
||||
} else if reserve < net_amount {
|
||||
let shortfall_msg = format!(
|
||||
"🚨 RESERVE SHORTFALL: custody holds {} USDT, need {} USDT to honor a pending {} USDT withdrawal from {}. \
|
||||
This must be reviewed by the signatories before any approval - do NOT approve as-is.",
|
||||
@@ -1462,9 +1503,9 @@ async fn process_withdrawal_block(
|
||||
match convert_ss58_prefix(&from_str, 0) {
|
||||
Ok(polkadot_recipient) => {
|
||||
let now = Utc::now();
|
||||
// Never attempt auto-pay against an already-known reserve shortfall -
|
||||
// that's exactly the case that must always go to human review.
|
||||
let auto_pay_attempt = if reserve < net_amount {
|
||||
// Never attempt auto-pay against an unverified or known-short reserve -
|
||||
// both cases must always go to human review.
|
||||
let auto_pay_attempt = if !reserve_known || reserve < net_amount {
|
||||
None
|
||||
} else {
|
||||
match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) {
|
||||
@@ -1560,8 +1601,12 @@ async fn process_withdrawal_block(
|
||||
" 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot",
|
||||
amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount
|
||||
);
|
||||
if reserve_known {
|
||||
info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
|
||||
reserve as f64 / 1_000_000.0);
|
||||
} else {
|
||||
info!(" Reserve could NOT be verified (Polkadot RPC unreachable at detection time) - signatories must check manually via pwap-web.");
|
||||
}
|
||||
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
@@ -1573,7 +1618,9 @@ async fn process_withdrawal_block(
|
||||
fee,
|
||||
net_amount,
|
||||
polkadot_tx_hash: None,
|
||||
status: if reserve < net_amount {
|
||||
status: if !reserve_known {
|
||||
"pending_multisig_approval_reserve_unverified".to_string()
|
||||
} else if reserve < net_amount {
|
||||
"pending_multisig_approval_reserve_shortfall".to_string()
|
||||
} else {
|
||||
"pending_multisig_approval".to_string()
|
||||
@@ -1822,7 +1869,7 @@ async fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
// Load or create config
|
||||
let config: BridgeConfig = if cli.config.exists() {
|
||||
let mut config: BridgeConfig = if cli.config.exists() {
|
||||
let content = std::fs::read_to_string(&cli.config)?;
|
||||
serde_json::from_str(&content)?
|
||||
} else {
|
||||
@@ -1833,6 +1880,17 @@ async fn main() -> Result<()> {
|
||||
default_config
|
||||
};
|
||||
|
||||
// Dedicated/API-keyed RPC endpoints (e.g. a Dwellir URL) carry a credential in the URL itself,
|
||||
// so - same reasoning as PEZBRIDGE_TELEGRAM_TOKEN/AUTOMATION_KEY_MNEMONIC below - they belong
|
||||
// in an env var backed by a tightly-permissioned EnvironmentFile, not in the git-tracked JSON
|
||||
// config. Unset means "use whatever bridge_config.json says" (the public endpoint by default).
|
||||
if let Ok(url) = std::env::var("POLKADOT_RPC_URL") {
|
||||
config.polkadot_rpc = url;
|
||||
}
|
||||
if let Ok(url) = std::env::var("PEZKUWI_RPC_URL") {
|
||||
config.pezkuwi_rpc = url;
|
||||
}
|
||||
|
||||
match cli.command {
|
||||
Commands::GenerateWallet { output } => {
|
||||
generate_wallet(&output)?;
|
||||
|
||||
Reference in new issue
Block a user