From 96840fd17e55d4e88c7697ad54cf6e7f7bad112e Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Tue, 14 Jul 2026 15:28:20 -0700 Subject: [PATCH] Add bounded automation-key auto-pay: per-tx cap, daily velocity cap, circuit breaker Small/mid-size swaps no longer have to wait on 3-of-5 for every single transaction. usdt-bridge can now execute Assets.transfer_approved directly via the automation key's existing delegated allowance when a deposit/withdrawal passes all three independent guards: - max_single_tx (50,000 USDT): any single transaction above this always queues for manual multisig review, no exceptions. - auto_pay_daily_cap (200,000 USDT): a rolling 24h budget tracked separately per leg (wUSDT payouts vs USDT payouts draw on separate on-chain allowances), not a per-tx ceiling - lets many legitimate sub-cap swaps clear same-day. - circuit breaker: if >=50% of the daily cap is consumed within 30 minutes, auto-pay halts entirely until a human runs reset-circuit-breaker - a burst that fast reads as a compromised key or a bug, not organic demand. The automation key can only ever move funds via its pre-approved allowance (never mint/burn/set_team/transfer_ownership - those stay multisig-only), so its worst case is bounded by whatever allowance + daily cap remain, not unbounded custody. Withdrawal auto-pay releases real USDT immediately but defers the corresponding wUSDT burn (Owner/Admin-only, no delegation exists for it) to a periodic manual multisig batch - tracked via a distinct auto_paid_burn_pending status so it can't be silently forgotten. Auto-pay is opt-in at the infra level: AUTOMATION_KEY_MNEMONIC unset means 100% manual behavior, unchanged from before. Bumped pezbridge-bot's renewal_threshold/topup_amount so the next 3-of-5-approved allowance renewal scales to the new daily cap. --- tools/usdt-bridge/bridge_config.json | 8 + tools/usdt-bridge/deploy/usdt-bridge.service | 60 ++ tools/usdt-bridge/pezbridge_bot_config.json | 4 +- tools/usdt-bridge/src/main.rs | 659 +++++++++++++++++-- 4 files changed, 661 insertions(+), 70 deletions(-) create mode 100644 tools/usdt-bridge/deploy/usdt-bridge.service diff --git a/tools/usdt-bridge/bridge_config.json b/tools/usdt-bridge/bridge_config.json index 9c72fc25..404f589a 100644 --- a/tools/usdt-bridge/bridge_config.json +++ b/tools/usdt-bridge/bridge_config.json @@ -12,6 +12,14 @@ "custody_multisig_pezkuwi": "5GvwxmCDp3PC33KHoeWSgj3S7ocE7nzk1jiCCZMPSDBFeNcj", "serok_chat_id": 5362991649, "ops_chat_id": -5420200114, + "max_single_tx": 50000000000, + "auto_pay_daily_cap": 200000000000, + "circuit_breaker_window_minutes": 30, + "circuit_breaker_fraction": 0.5, + "_automation_2026-07-14": { + "note": "Small/mid swaps auto-pay via the automation key's Assets.transfer_approved allowance - no per-transaction 3-of-5 wait. max_single_tx (50,000 USDT) caps any single auto-paid transaction; auto_pay_daily_cap (200,000 USDT) is a rolling 24h budget tracked separately per leg (deposit wUSDT payouts vs withdrawal USDT payouts), NOT a per-tx ceiling - many sub-50K swaps can clear same-day. circuit_breaker_fraction/circuit_breaker_window_minutes halt all auto-pay if the daily budget is consumed abnormally fast (>=50% within 30 min) until a human runs `reset-circuit-breaker`. Anything over max_single_tx, or that would exceed the remaining daily budget, or while the breaker is tripped, ALWAYS falls back to full manual 3-of-5 review via pwap-web - this never bypasses the multisig, it only decides whether a given event needs it. Requires AUTOMATION_KEY_MNEMONIC env var to be set (see res/bridge_automation_key.json) - if unset, behavior is 100% manual, unchanged from before this feature.", + "requires_on_chain_action": "The automation key's actual Assets.approve_transfer allowance is renewed by PezbridgeBot's 3-of-5-signed renewal flow (pezbridge_bot_config.json's renewal_threshold/topup_amount) - those were bumped alongside this to top up to a size that supports the new daily cap. The NEXT renewal signers approve via pwap-web will lift the real on-chain ceiling; until then, auto-pay is bounded by whatever allowance already exists on-chain today, which is smaller. This is a fail-safe direction: if the on-chain allowance runs out, the system simply falls back to manual, never fails open." + }, "_custody_migration_2026-07-13": { "note": "Custody is no longer a single hot key. wUSDT (asset 1000) owner/issuer/admin/freezer on Pezkuwi Asset Hub is now the 3-of-5 multisig below - seed_path/BRIDGE_SEED_MNEMONIC in this file is stale and must not be used to mint/burn/transfer_ownership going forward. listen_deposits/listen_withdrawals in src/main.rs were updated the same day: they now only detect events and record status=pending_multisig_approval - actual mint/burn/release requires 3 of the 5 signatories to approve via pwap-web's multisig UI (shared/lib/multisig.ts, shared/lib/usdt.ts).", "multisig_threshold": 3, diff --git a/tools/usdt-bridge/deploy/usdt-bridge.service b/tools/usdt-bridge/deploy/usdt-bridge.service new file mode 100644 index 00000000..9c98929d --- /dev/null +++ b/tools/usdt-bridge/deploy/usdt-bridge.service @@ -0,0 +1,60 @@ +[Unit] +Description=usdt-bridge listen-all - USDT/wUSDT listener; auto-pays sub-cap swaps via the bounded automation-key allowance, everything else queues for 3-of-5 +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=usdtbridge +Group=usdtbridge +WorkingDirectory=/opt/usdtbridge/data +ExecStart=/opt/usdtbridge/bin/usdt-bridge --config /opt/usdtbridge/bin/bridge_config.json listen-all +Environment=RUST_LOG=info +EnvironmentFile=/etc/pezbridgebot/telegram_token.env +EnvironmentFile=-/etc/usdtbridge/automation_key.env +Restart=on-failure +RestartSec=10 + +# --- Hardening --- +# Deployed 2026-07-14 on NEW-10 (89.116.25.180), same UTIL bucket as pezbridge-bot (no +# consensus/fund-critical services here - see dks-ci-runners/vps2-dev-decommission-status +# memory for the placement convention). UPDATED 2026-07-14: this process now also loads the +# automation key's mnemonic (from a separate, tighter-scoped EnvironmentFile than the Telegram +# token) and can sign+submit Assets.transfer_approved on both chains - but ONLY within the +# multisig's pre-approved delegated allowance, capped per-tx (max_single_tx) and per rolling 24h +# window (auto_pay_daily_cap) with a circuit breaker on burst consumption. It can never mint, +# burn, set_team, or transfer_ownership - those stay exclusively multisig (3-of-5), same as +# before. Worst case of this key being compromised is bounded by whatever allowance + daily cap +# remain at the time, not unbounded custody - a materially different (and much smaller) risk +# tier than the legacy single-key bot this whole migration replaced. Hardening kept as-is; the +# automation_key.env file itself is root:usdtbridge 0640, separate from the Telegram token file +# so a Telegram-token leak alone can never expose the signing key. The leading "-" on that +# EnvironmentFile line is required (not decorative): the file does not exist until an operator +# deliberately provisions it, and the unit must still start cleanly without it, matching the +# code's own AUTOMATION_KEY_MNEMONIC-unset fallback (100% manual, auto-pay simply inactive). +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +RestrictSUIDSGID=yes +RemoveIPC=yes +LockPersonality=yes +RestrictRealtime=yes +RestrictNamespaces=yes +MemoryDenyWriteExecute=yes +SystemCallFilter=@system-service +SystemCallArchitectures=native +CapabilityBoundingSet= +AmbientCapabilities= +ReadWritePaths=/opt/usdtbridge/data +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/tools/usdt-bridge/pezbridge_bot_config.json b/tools/usdt-bridge/pezbridge_bot_config.json index 7395de2d..da445f64 100644 --- a/tools/usdt-bridge/pezbridge_bot_config.json +++ b/tools/usdt-bridge/pezbridge_bot_config.json @@ -3,7 +3,7 @@ "serok_chat_id": 5362991649, "ops_chat_id": -5420200114, "poll_interval_secs": 600, - "renewal_threshold": 3000000000, - "topup_amount": 10000000000, + "renewal_threshold": 40000000000, + "topup_amount": 200000000000, "state_path": "/opt/pezbridgebot/data/pezbridge_bot_state.json" } diff --git a/tools/usdt-bridge/src/main.rs b/tools/usdt-bridge/src/main.rs index d0264bdf..5aa4b468 100644 --- a/tools/usdt-bridge/src/main.rs +++ b/tools/usdt-bridge/src/main.rs @@ -27,6 +27,7 @@ //! - Fees: Configurable (default 0.1%) use anyhow::{anyhow, Context, Result}; +use chrono::{DateTime, Utc}; use clap::{Parser, Subcommand}; use serde::{Deserialize, Serialize}; use sp_core::{crypto::Ss58Codec, sr25519, Pair}; @@ -34,9 +35,29 @@ use std::path::PathBuf; use std::sync::LazyLock; use subxt::{OnlineClient, SubstrateConfig}; use subxt::dynamic::{At, Value}; +use subxt_signer::bip39::Mnemonic; +use subxt_signer::sr25519::Keypair as PolkadotKeypair; +// Pezkuwi-side signing MUST go through this fork, not vanilla subxt - vanilla subxt's default +// extrinsic params don't match Pezkuwi's signed-extension/hasher setup and produce "bad +// signature" errors on submission (see examples/migrate_wusdt_to_multisig.rs, which hit this +// first). Aliased to avoid clashing with the vanilla `Value`/`OnlineClient` already imported +// above and used throughout this file for read-only queries on both chains. +use pezkuwi_subxt::dynamic::Value as PValue; +use pezkuwi_subxt::{OnlineClient as PezkuwiSigningClient, PezkuwiConfig}; +use pezkuwi_subxt_signer::bip39::Mnemonic as PezkuwiMnemonic; +use pezkuwi_subxt_signer::sr25519::Keypair as PezkuwiKeypair; use tokio::sync::Mutex as AsyncMutex; use tracing::{info, warn, error}; +// The automation key is a deliberately limited, server-held delegate: it can only ever move +// funds the multisig has explicitly pre-approved via `Assets.approve_transfer`, up to whatever +// allowance remains, and only via `transfer_approved` (never mint/burn/set_team/transfer_ownership +// - those stay Owner/Admin-only, i.e. multisig-only, forever). This is categorically different +// from the 5 signatories' own seeds - see the standing rule in this session: never load a +// signatory's seed programmatically to bypass human multisig approval. This key exists +// specifically so it CAN be loaded programmatically; that is its whole purpose. +const AUTOMATION_KEY_ADDRESS: &str = "5GQu4PFUb1f3MTJ7i7c1CtLgDk3TVvpSW1VbQCRmfkMoC8cM"; + // ============================================================================ // Configuration // ============================================================================ @@ -62,11 +83,27 @@ pub struct BridgeConfig { pub seed_path: PathBuf, /// Database path pub db_path: PathBuf, - /// Maximum amount processed automatically in a single deposit/withdrawal (base units). - /// Anything above this is left unprocessed for manual review - caps the blast radius - /// of a single bad event (bug, reorg, compromised sender) instead of relaying it blindly. + /// Per-transaction ceiling the automation key may execute on its own (base units). Anything + /// above this ALWAYS queues for manual 3-of-5 review, regardless of remaining daily budget - + /// caps the blast radius of a single bad event (bug, reorg, compromised sender, compromised + /// automation key) to one transaction's worth, no matter how much daily headroom exists. #[serde(default = "default_max_single_tx")] pub max_single_tx: u128, + /// Rolling 24h ceiling on total value the automation key may auto-pay out, tracked + /// independently per leg (deposit payouts in wUSDT vs withdrawal payouts in USDT draw on + /// separate on-chain allowances on separate chains). Bounds worst-case exposure from a + /// compromised automation key to one day's volume rather than an unbounded amount, while + /// still letting many legitimate sub-cap swaps clear same-day without human involvement. + #[serde(default = "default_auto_pay_daily_cap")] + pub auto_pay_daily_cap: u128, + /// Circuit-breaker window: if `circuit_breaker_fraction` of the daily cap is consumed within + /// this many minutes, auto-pay halts entirely (falls back to manual review) until a human + /// clears it via the `reset-circuit-breaker` command - a burst that fast is far more + /// consistent with a compromised key or a bug than organic demand. + #[serde(default = "default_circuit_breaker_window_minutes")] + pub circuit_breaker_window_minutes: i64, + #[serde(default = "default_circuit_breaker_fraction")] + pub circuit_breaker_fraction: f64, /// The 3-of-5 multisig custody account, Polkadot Asset Hub SS58 encoding. Deposits are /// watched for transfers TO this address - it replaced the old single-key custody address. #[serde(default = "default_multisig_polkadot")] @@ -85,7 +122,7 @@ pub struct BridgeConfig { } fn default_max_single_tx() -> u128 { - 5_000_000_000 // 5,000 USDT (6 decimals) - conservative pilot-stage cap + 50_000_000_000 // 50,000 USDT (6 decimals) - small/mid-size swaps clear without waiting on 3-of-5 } fn default_multisig_polkadot() -> String { @@ -96,6 +133,18 @@ fn default_multisig_pezkuwi() -> String { "5GvwxmCDp3PC33KHoeWSgj3S7ocE7nzk1jiCCZMPSDBFeNcj".to_string() } +fn default_auto_pay_daily_cap() -> u128 { + 200_000_000_000 // 200,000 USDT/day - rolling 24h automation budget, per leg +} + +fn default_circuit_breaker_window_minutes() -> i64 { + 30 +} + +fn default_circuit_breaker_fraction() -> f64 { + 0.5 // >=50% of the daily cap consumed within the window trips the breaker +} + impl Default for BridgeConfig { fn default() -> Self { Self { @@ -113,6 +162,9 @@ impl Default for BridgeConfig { custody_multisig_pezkuwi: default_multisig_pezkuwi(), serok_chat_id: 0, ops_chat_id: 0, + auto_pay_daily_cap: default_auto_pay_daily_cap(), + circuit_breaker_window_minutes: default_circuit_breaker_window_minutes(), + circuit_breaker_fraction: default_circuit_breaker_fraction(), } } } @@ -172,6 +224,13 @@ enum Commands { /// Check balances on both chains Balances, + + /// Show automation-key auto-pay status: rolling 24h volume per leg vs caps, and whether + /// the circuit breaker is tripped + AutomationStatus, + + /// Clear a tripped circuit breaker after human review - required before auto-pay resumes + ResetCircuitBreaker, } // ============================================================================ @@ -279,6 +338,29 @@ struct BridgeDatabase { deposits: Vec, withdrawals: Vec, stats: BridgeStats, + /// Ledger of automation-key auto-payouts, used to compute rolling-window volume per leg. + /// Pruned to the last 48h on every write - only ever needs a 24h lookback plus slack. + #[serde(default)] + auto_payouts: Vec, + #[serde(default)] + circuit_breaker_tripped: bool, + #[serde(default)] + circuit_breaker_reason: Option, + #[serde(default)] + circuit_breaker_tripped_at: Option, +} + +/// One leg's identity for rolling-volume tracking - deposit payouts (wUSDT, Pezkuwi side) and +/// withdrawal payouts (USDT, Polkadot side) draw on separate on-chain allowances and must never +/// share a budget. +const LEG_DEPOSIT_WUSDT_PAYOUT: &str = "deposit_wusdt_payout"; +const LEG_WITHDRAWAL_USDT_PAYOUT: &str = "withdrawal_usdt_payout"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct AutoPayoutRecord { + leg: String, + amount: u128, + at: String, // RFC3339 } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -375,6 +457,37 @@ impl BridgeDatabase { fn is_withdrawal_processed(&self, pezkuwi_tx_hash: &str) -> bool { self.withdrawals.iter().any(|w| w.pezkuwi_tx_hash == pezkuwi_tx_hash) } + + /// Sum of auto-payouts on `leg` within the last `window_minutes` minutes, relative to `now`. + /// Used for both the 24h daily-cap check (window_minutes = 1440) and the fast-burst + /// circuit-breaker check (window_minutes = config.circuit_breaker_window_minutes). + fn recent_auto_payout_volume(&self, leg: &str, now: DateTime, window_minutes: i64) -> u128 { + self.auto_payouts + .iter() + .filter(|p| p.leg == leg) + .filter_map(|p| DateTime::parse_from_rfc3339(&p.at).ok().map(|t| (t.with_timezone(&Utc), p.amount))) + .filter(|(t, _)| now.signed_duration_since(*t) < chrono::Duration::minutes(window_minutes)) + .map(|(_, a)| a) + .sum() + } + + /// Records a completed auto-payout and prunes entries older than 48h (only a 24h lookback is + /// ever needed - the extra slack just avoids pruning right at the boundary of a live check). + fn record_auto_payout(&mut self, leg: &str, amount: u128, now: DateTime) { + self.auto_payouts.push(AutoPayoutRecord { leg: leg.to_string(), amount, at: now.to_rfc3339() }); + let cutoff = now - chrono::Duration::hours(48); + self.auto_payouts.retain(|p| { + DateTime::parse_from_rfc3339(&p.at) + .map(|t| t.with_timezone(&Utc) > cutoff) + .unwrap_or(true) // keep anything unparseable rather than silently drop history + }); + } + + fn trip_circuit_breaker(&mut self, reason: String, now: DateTime) { + self.circuit_breaker_tripped = true; + self.circuit_breaker_reason = Some(reason); + self.circuit_breaker_tripped_at = Some(now.to_rfc3339()); + } } /// Guards every load-mutate-save critical section on `BridgeDatabase`. `listen_deposits` and @@ -509,6 +622,213 @@ async fn notify_telegram(config: &BridgeConfig, msg: &str) { } } +// ============================================================================ +// Automation key: bounded auto-pay via Assets.transfer_approved +// +// The automation key holds ONLY a delegated `Assets.approve_transfer` allowance from the +// multisig - it can never mint, burn, set_team, or transfer_ownership. Its worst case (private +// key stolen) is bounded by whatever allowance remains on-chain at the time, which in turn is +// bounded by `max_single_tx` per transaction and kept small by design (renewed in modest +// top-ups, requiring fresh 3-of-5 approval each time - see pezbridge_bot.rs). This module adds +// two more independent layers on top of that on-chain bound: a rolling 24h volume cap per leg, +// and a burst-rate circuit breaker that halts all auto-pay if that cap is consumed too fast. +// ============================================================================ + +/// Both chain-specific signing keypairs for the SAME automation key (same seed, same sr25519 +/// key - just wrapped in the two different subxt fork types each chain's extrinsic params need). +/// `None` fields mean the corresponding on-chain client failed to connect at startup; auto-pay +/// on that leg is then simply skipped (falls back to today's fully-manual pending-record path). +struct AutomationContext { + pezkuwi_signing_client: Option>, + pezkuwi_keypair: Option, + polkadot_signing_client: Option>, + polkadot_keypair: Option, +} + +impl AutomationContext { + fn disabled() -> Self { + Self { pezkuwi_signing_client: None, pezkuwi_keypair: None, polkadot_signing_client: None, polkadot_keypair: None } + } +} + +/// Loads the automation key's mnemonic from `AUTOMATION_KEY_MNEMONIC` and connects signing +/// clients on both chains. If the env var isn't set, returns a fully-disabled context - the +/// system behaves exactly as it did before this feature existed (100% manual multisig review). +/// This is deliberate: auto-pay must never be "on by default" just because the binary was +/// rebuilt: it activates only once an operator explicitly provisions the credential. +async fn init_automation_context(config: &BridgeConfig) -> AutomationContext { + let Ok(mnemonic_str) = std::env::var("AUTOMATION_KEY_MNEMONIC") else { + warn!("AUTOMATION_KEY_MNEMONIC not set - auto-pay disabled, all deposits/withdrawals will queue for manual 3-of-5 review (this is safe, just fully manual)"); + return AutomationContext::disabled(); + }; + + let pezkuwi_keypair = (|| -> Result { + let m = PezkuwiMnemonic::parse(&mnemonic_str).context("parsing automation mnemonic (pezkuwi)")?; + let kp = PezkuwiKeypair::from_phrase(&m, None).context("deriving pezkuwi keypair")?; + let addr = kp.public_key().to_account_id().to_string(); + if addr != AUTOMATION_KEY_ADDRESS { + return Err(anyhow!("AUTOMATION_KEY_MNEMONIC derives {addr}, expected {AUTOMATION_KEY_ADDRESS} - refusing to use a mismatched key")); + } + Ok(kp) + })(); + + let polkadot_keypair = (|| -> Result { + let m = Mnemonic::parse(&mnemonic_str).context("parsing automation mnemonic (polkadot)")?; + PolkadotKeypair::from_phrase(&m, None).context("deriving polkadot keypair") + })(); + + let (pezkuwi_keypair, polkadot_keypair) = match (pezkuwi_keypair, polkadot_keypair) { + (Ok(p), Ok(d)) => (Some(p), Some(d)), + (Err(e), _) | (_, Err(e)) => { + error!("Failed to derive automation keypair(s), auto-pay disabled: {e:#}"); + return AutomationContext::disabled(); + } + }; + + let pezkuwi_signing_client = match PezkuwiSigningClient::::from_insecure_url(&config.pezkuwi_rpc).await { + Ok(c) => Some(c), + Err(e) => { error!("Auto-pay (deposit/wUSDT leg) disabled - failed to connect signing client: {e:#}"); None } + }; + let polkadot_signing_client = match OnlineClient::::from_url(&config.polkadot_rpc).await { + Ok(c) => Some(c), + Err(e) => { error!("Auto-pay (withdrawal/USDT leg) disabled - failed to connect signing client: {e:#}"); None } + }; + + info!("Automation key loaded ({AUTOMATION_KEY_ADDRESS}) - auto-pay ENABLED where signing clients connected"); + AutomationContext { pezkuwi_signing_client, pezkuwi_keypair, polkadot_signing_client, polkadot_keypair } +} + +fn multi_address_id_pezkuwi(bytes: [u8; 32]) -> PValue { + PValue::unnamed_variant("Id", [PValue::from_bytes(bytes)]) +} + +fn multi_address_id_polkadot(bytes: [u8; 32]) -> Value { + Value::unnamed_variant("Id", [Value::from_bytes(bytes)]) +} + +/// Submits `Assets.transfer_approved(asset_id, owner, destination, amount)` on Pezkuwi Asset Hub, +/// signed by the automation key, and waits for on-chain dispatch success. Used for the +/// USDT->wUSDT deposit leg (paying wUSDT out of the multisig's seed pool). +async fn execute_wusdt_auto_payout( + client: &PezkuwiSigningClient, + keypair: &PezkuwiKeypair, + asset_id: u32, + owner_bytes: [u8; 32], + destination_bytes: [u8; 32], + amount: u128, +) -> Result { + let call = pezkuwi_subxt::dynamic::tx( + "Assets", + "transfer_approved", + vec![ + PValue::primitive(asset_id.into()), + multi_address_id_pezkuwi(owner_bytes), + multi_address_id_pezkuwi(destination_bytes), + PValue::u128(amount), + ], + ); + let mut progress = client.tx().sign_and_submit_then_watch_default(&call, keypair).await + .context("submitting transfer_approved (pezkuwi)")?; + let hash = format!("0x{}", hex::encode(progress.extrinsic_hash().as_ref())); + use pezkuwi_subxt::tx::TxStatus; + loop { + match progress.next().await { + Some(Ok(TxStatus::InBestBlock(details))) => { + details.wait_for_success().await.map_err(|e| anyhow!("transfer_approved dispatch failed: {e}"))?; + return Ok(hash); + } + Some(Ok(TxStatus::Error { message })) => return Err(anyhow!("tx error: {message}")), + Some(Ok(TxStatus::Invalid { message })) => return Err(anyhow!("tx invalid: {message}")), + Some(Ok(TxStatus::Dropped { message })) => return Err(anyhow!("tx dropped: {message}")), + Some(Err(e)) => return Err(anyhow!("stream error: {e}")), + None => return Err(anyhow!("stream ended without a result")), + _ => continue, + } + } +} + +/// Same as `execute_wusdt_auto_payout` but on Polkadot Asset Hub via vanilla subxt (correct here +/// - the bad-signature gotcha only applies to Pezkuwi's custom chain). Used for the wUSDT->USDT +/// withdrawal leg (paying real USDT out of the multisig's Polkadot-side reserve). +async fn execute_usdt_auto_payout( + client: &OnlineClient, + keypair: &PolkadotKeypair, + asset_id: u32, + owner_bytes: [u8; 32], + destination_bytes: [u8; 32], + amount: u128, +) -> Result { + let call = subxt::dynamic::tx( + "Assets", + "transfer_approved", + vec![ + Value::primitive(asset_id.into()), + multi_address_id_polkadot(owner_bytes), + multi_address_id_polkadot(destination_bytes), + Value::u128(amount), + ], + ); + let mut progress = client.tx().sign_and_submit_then_watch_default(&call, keypair).await + .context("submitting transfer_approved (polkadot)")?; + let hash = format!("0x{}", hex::encode(progress.extrinsic_hash().as_ref())); + use subxt::tx::TxStatus; + loop { + match progress.next().await { + Some(Ok(TxStatus::InBestBlock(details))) => { + details.wait_for_success().await.map_err(|e| anyhow!("transfer_approved dispatch failed: {e}"))?; + return Ok(hash); + } + Some(Ok(TxStatus::Error { message })) => return Err(anyhow!("tx error: {message}")), + Some(Ok(TxStatus::Invalid { message })) => return Err(anyhow!("tx invalid: {message}")), + Some(Ok(TxStatus::Dropped { message })) => return Err(anyhow!("tx dropped: {message}")), + Some(Err(e)) => return Err(anyhow!("stream error: {e}")), + None => return Err(anyhow!("stream ended without a result")), + _ => continue, + } + } +} + +/// Shared eligibility gate for both legs: circuit breaker must be clear, amount within per-tx +/// cap (redundant with the caller's existing max_single_tx check, checked again here so this +/// function is safe to call from anywhere), and (existing + amount) must stay within the +/// rolling 24h cap for this leg. Returns the disqualifying reason as Err(reason) for logging/ +/// alerting, or Ok(()) if auto-pay should proceed. +fn check_auto_pay_eligible(db: &BridgeDatabase, config: &BridgeConfig, leg: &str, amount: u128, now: DateTime) -> std::result::Result<(), String> { + if db.circuit_breaker_tripped { + return Err(format!("circuit breaker is tripped ({}) - auto-pay halted until a human runs reset-circuit-breaker", db.circuit_breaker_reason.as_deref().unwrap_or("no reason recorded"))); + } + if amount > config.max_single_tx { + return Err(format!("amount {} exceeds max_single_tx {}", amount, config.max_single_tx)); + } + let existing = db.recent_auto_payout_volume(leg, now, 24 * 60); + if existing + amount > config.auto_pay_daily_cap { + return Err(format!( + "would exceed daily auto-pay cap for {leg}: {} already used + {} requested > {} cap", + existing, amount, config.auto_pay_daily_cap + )); + } + Ok(()) +} + +/// After a successful auto-payout, checks whether the burst-rate circuit breaker should trip - +/// i.e. whether `circuit_breaker_fraction` of the daily cap was consumed within +/// `circuit_breaker_window_minutes`. Call this AFTER `record_auto_payout` so the just-completed +/// payout is included in the window sum. +fn maybe_trip_circuit_breaker(db: &mut BridgeDatabase, config: &BridgeConfig, leg: &str, now: DateTime) -> Option { + let recent = db.recent_auto_payout_volume(leg, now, config.circuit_breaker_window_minutes); + let threshold = (config.auto_pay_daily_cap as f64 * config.circuit_breaker_fraction) as u128; + if recent >= threshold { + let reason = format!( + "{leg}: {} auto-paid within {} min (>= {:.0}% of daily cap {}) - burst rate far exceeds normal organic demand", + recent, config.circuit_breaker_window_minutes, config.circuit_breaker_fraction * 100.0, config.auto_pay_daily_cap + ); + db.trip_circuit_breaker(reason.clone(), now); + Some(reason) + } else { + None + } +} + // ============================================================================ // Bridge Operations // ============================================================================ @@ -662,7 +982,7 @@ fn calculate_fee(amount: u128, fee_basis_points: u32) -> u128 { amount * fee_basis_points as u128 / 10_000 } -async fn listen_deposits(config: &BridgeConfig) -> Result<()> { +async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext) -> Result<()> { let polkadot_addr = &config.custody_multisig_polkadot; println!("╔══════════════════════════════════════════════════════════════╗"); @@ -721,10 +1041,8 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> { if to_bytes.is_some() && to_bytes == target_bytes { { - let from_str = from_field - .and_then(account_id_from_dynamic) - .map(|b| account_bytes_to_ss58(b, 0)) - .unwrap_or_default(); + let from_bytes_opt = from_field.and_then(account_id_from_dynamic); + let from_str = from_bytes_opt.map(|b| account_bytes_to_ss58(b, 0)).unwrap_or_default(); let tx_hash = format!("block-{}-{}", block_number, from_str); info!("📥 DEPOSIT DETECTED!"); @@ -756,32 +1074,99 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> { // correct rather than a placeholder. match convert_ss58_prefix(&from_str, 42) { Ok(pezkuwi_recipient) => { - info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-minting."); - info!( - " 3 of 5 signatories must approve: Assets.transfer({}, {}, {}) from the multisig's seed pool", - config.pezkuwi_wusdt_asset_id, pezkuwi_recipient, net_amount - ); - info!(" (fee: {} USDT, net: {} USDT) - use pwap-web's multisig UI.", - fee as f64 / 1_000_000.0, net_amount as f64 / 1_000_000.0); + let now = Utc::now(); + let auto_pay_attempt = match (from_bytes_opt, &automation.pezkuwi_signing_client, &automation.pezkuwi_keypair) { + (Some(dest_bytes), Some(client), Some(keypair)) => { + match check_auto_pay_eligible(&db, config, LEG_DEPOSIT_WUSDT_PAYOUT, net_amount, now) { + Ok(()) => { + let owner_bytes = sp_core::crypto::AccountId32::from_ss58check(&config.custody_multisig_pezkuwi) + .map(Into::<[u8; 32]>::into) + .map_err(|e| format!("bad custody_multisig_pezkuwi config: {e:?}")); + match owner_bytes { + Ok(owner_bytes) => Some(execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await), + Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } + } + } + Err(reason) => { info!(" ℹ️ Not auto-paying (falls back to manual): {reason}"); None } + } + } + _ => None, // automation not configured/connected - always-manual fallback + }; - db.add_deposit(DepositRecord { - id: db.next_deposit_id(), - polkadot_tx_hash: tx_hash.clone(), - polkadot_block: block_number as u64, - sender_address: from_str.clone(), - amount, - fee, - net_amount, - pezkuwi_tx_hash: None, - status: "pending_multisig_approval".to_string(), - created_at: chrono::Utc::now().to_rfc3339(), - processed_at: None, - }); - db.save(&config.db_path)?; - notify_telegram(config, &format!( - "📥 New deposit pending approval: {} USDT from {} -> mint {} wUSDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.", - amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient - )).await; + match auto_pay_attempt { + Some(Ok(pezkuwi_tx_hash)) => { + info!(" ✅ AUTO-PAID: {} wUSDT -> {} (tx {})", net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash); + db.record_auto_payout(LEG_DEPOSIT_WUSDT_PAYOUT, net_amount, now); + let breaker_tripped = maybe_trip_circuit_breaker(&mut db, config, LEG_DEPOSIT_WUSDT_PAYOUT, now); + db.add_deposit(DepositRecord { + id: db.next_deposit_id(), + polkadot_tx_hash: tx_hash.clone(), + polkadot_block: block_number as u64, + sender_address: from_str.clone(), + amount, fee, net_amount, + pezkuwi_tx_hash: Some(pezkuwi_tx_hash.clone()), + status: "auto_completed".to_string(), + created_at: now.to_rfc3339(), + processed_at: Some(now.to_rfc3339()), + }); + db.save(&config.db_path)?; + if let Some(reason) = breaker_tripped { + notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (deposit/wUSDT leg): {reason}. All further deposits now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await; + } else { + notify_telegram(config, &format!( + "✅ Auto-paid deposit: {} USDT from {} -> {} wUSDT (net of fee) to {}. tx {}", + amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash + )).await; + } + } + Some(Err(e)) => { + error!(" ❌ Auto-pay execution FAILED, falling back to manual review: {e:#}"); + notify_telegram(config, &format!( + "⚠️ Auto-pay attempt failed for deposit {} USDT from {} - falling back to manual 3-of-5 review. Error: {e:#}", + amount as f64 / 1_000_000.0, from_str + )).await; + db.add_deposit(DepositRecord { + id: db.next_deposit_id(), + polkadot_tx_hash: tx_hash.clone(), + polkadot_block: block_number as u64, + sender_address: from_str.clone(), + amount, fee, net_amount, + pezkuwi_tx_hash: None, + status: "pending_multisig_approval".to_string(), + created_at: now.to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; + } + None => { + info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-minting."); + info!( + " 3 of 5 signatories must approve: Assets.transfer({}, {}, {}) from the multisig's seed pool", + config.pezkuwi_wusdt_asset_id, pezkuwi_recipient, net_amount + ); + info!(" (fee: {} USDT, net: {} USDT) - use pwap-web's multisig UI.", + fee as f64 / 1_000_000.0, net_amount as f64 / 1_000_000.0); + + db.add_deposit(DepositRecord { + id: db.next_deposit_id(), + polkadot_tx_hash: tx_hash.clone(), + polkadot_block: block_number as u64, + sender_address: from_str.clone(), + amount, + fee, + net_amount, + pezkuwi_tx_hash: None, + status: "pending_multisig_approval".to_string(), + created_at: now.to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; + notify_telegram(config, &format!( + "📥 New deposit pending approval: {} USDT from {} -> mint {} wUSDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.", + amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient + )).await; + } + } } Err(e) => { error!(" ❌ Could not derive Pezkuwi recipient from sender {}: {}", from_str, e); @@ -807,7 +1192,7 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> { /// bridge was missing - a mismatch here means the peg is under-collateralized and must halt, /// not silently short-pay a user), then release on Polkadot. Mirrors listen_deposits() and /// shares its event-matching caveat - see the comment there. -async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> { +async fn listen_withdrawals(config: &BridgeConfig, automation: &AutomationContext) -> Result<()> { let polkadot_bridge_addr = &config.custody_multisig_polkadot; let pezkuwi_bridge_addr = &config.custody_multisig_pezkuwi; @@ -857,10 +1242,8 @@ async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> { if to_bytes.is_some() && to_bytes == target_bytes { { - let from_str = from_field - .and_then(account_id_from_dynamic) - .map(|b| account_bytes_to_ss58(b, 42)) - .unwrap_or_default(); + let from_bytes_opt = from_field.and_then(account_id_from_dynamic); + let from_str = from_bytes_opt.map(|b| account_bytes_to_ss58(b, 42)).unwrap_or_default(); let tx_hash = format!("block-{}-{}", block_number, from_str); info!("📤 WITHDRAWAL DETECTED!"); @@ -929,33 +1312,114 @@ async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> { match convert_ss58_prefix(&from_str, 0) { Ok(polkadot_recipient) => { - info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-burning/releasing."); - info!( - " 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot", - amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount - ); - info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.", - reserve as f64 / 1_000_000.0); + let now = Utc::now(); + // Never attempt auto-pay against an already-known reserve shortfall - + // that's exactly the case that must always go to human review. + let auto_pay_attempt = if reserve < net_amount { + None + } else { + match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) { + (Some(dest_bytes), Some(client), Some(keypair)) => { + match check_auto_pay_eligible(&db, config, LEG_WITHDRAWAL_USDT_PAYOUT, net_amount, now) { + Ok(()) => { + let owner_bytes = sp_core::crypto::AccountId32::from_ss58check(polkadot_bridge_addr) + .map(Into::<[u8; 32]>::into) + .map_err(|e| format!("bad custody_multisig_polkadot config: {e:?}")); + match owner_bytes { + Ok(owner_bytes) => Some(execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await), + Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } + } + } + Err(reason) => { info!(" ℹ️ Not auto-paying (falls back to manual): {reason}"); None } + } + } + _ => None, + } + }; - db.add_withdrawal(WithdrawalRecord { - id: db.next_withdrawal_id(), - pezkuwi_tx_hash: tx_hash.clone(), - pezkuwi_block: block_number as u64, - sender_address: from_str.clone(), - destination_address: polkadot_recipient, - amount, - fee, - net_amount, - polkadot_tx_hash: None, - status: if reserve < net_amount { - "pending_multisig_approval_reserve_shortfall".to_string() - } else { - "pending_multisig_approval".to_string() - }, - created_at: chrono::Utc::now().to_rfc3339(), - processed_at: None, - }); - db.save(&config.db_path)?; + match auto_pay_attempt { + Some(Ok(polkadot_tx_hash)) => { + // The wUSDT the user sent in is NOT burned here - burn is + // Owner/Admin-only (multisig-only, no delegation exists for + // it). Paying the user out is what's time-critical; the + // matching burn is deferred bookkeeping the signatories batch + // periodically via pwap-web - tracked explicitly by this + // status so it can never be silently forgotten. + info!(" ✅ AUTO-PAID: {} USDT -> {} (tx {}); wUSDT burn still pending (multisig-only)", net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash); + db.record_auto_payout(LEG_WITHDRAWAL_USDT_PAYOUT, net_amount, now); + let breaker_tripped = maybe_trip_circuit_breaker(&mut db, config, LEG_WITHDRAWAL_USDT_PAYOUT, now); + db.add_withdrawal(WithdrawalRecord { + id: db.next_withdrawal_id(), + pezkuwi_tx_hash: tx_hash.clone(), + pezkuwi_block: block_number as u64, + sender_address: from_str.clone(), + destination_address: polkadot_recipient.clone(), + amount, fee, net_amount, + polkadot_tx_hash: Some(polkadot_tx_hash.clone()), + status: "auto_paid_burn_pending".to_string(), + created_at: now.to_rfc3339(), + processed_at: Some(now.to_rfc3339()), + }); + db.save(&config.db_path)?; + if let Some(reason) = breaker_tripped { + notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (withdrawal/USDT leg): {reason}. All further withdrawals now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await; + } else { + notify_telegram(config, &format!( + "✅ Auto-paid withdrawal: {} wUSDT from {} -> {} USDT (net of fee) to {}. tx {}. wUSDT burn still pending - batch via pwap-web when convenient.", + amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash + )).await; + } + } + Some(Err(e)) => { + error!(" ❌ Auto-pay execution FAILED, falling back to manual review: {e:#}"); + notify_telegram(config, &format!( + "⚠️ Auto-pay attempt failed for withdrawal {} USDT from {} - falling back to manual 3-of-5 review. Error: {e:#}", + amount as f64 / 1_000_000.0, from_str + )).await; + db.add_withdrawal(WithdrawalRecord { + id: db.next_withdrawal_id(), + pezkuwi_tx_hash: tx_hash.clone(), + pezkuwi_block: block_number as u64, + sender_address: from_str.clone(), + destination_address: polkadot_recipient, + amount, fee, net_amount, + polkadot_tx_hash: None, + status: "pending_multisig_approval".to_string(), + created_at: now.to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; + } + None => { + info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-burning/releasing."); + info!( + " 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot", + amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount + ); + info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.", + reserve as f64 / 1_000_000.0); + + db.add_withdrawal(WithdrawalRecord { + id: db.next_withdrawal_id(), + pezkuwi_tx_hash: tx_hash.clone(), + pezkuwi_block: block_number as u64, + sender_address: from_str.clone(), + destination_address: polkadot_recipient, + amount, + fee, + net_amount, + polkadot_tx_hash: None, + status: if reserve < net_amount { + "pending_multisig_approval_reserve_shortfall".to_string() + } else { + "pending_multisig_approval".to_string() + }, + created_at: now.to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; + } + } } Err(e) => { error!(" ❌ Could not derive Polkadot recipient from sender {}: {}", from_str, e); @@ -1000,7 +1464,58 @@ async fn process_withdrawals(config: &BridgeConfig) -> Result<()> { } async fn listen_all(config: &BridgeConfig) -> Result<()> { - tokio::try_join!(listen_deposits(config), listen_withdrawals(config))?; + let automation = init_automation_context(config).await; + tokio::try_join!(listen_deposits(config, &automation), listen_withdrawals(config, &automation))?; + Ok(()) +} + +async fn show_automation_status(config: &BridgeConfig) -> Result<()> { + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ AUTOMATION KEY AUTO-PAY STATUS ║"); + println!("╚══════════════════════════════════════════════════════════════╝\n"); + + println!("Per-tx cap (max_single_tx): {} USDT", config.max_single_tx as f64 / 1_000_000.0); + println!("Daily cap (per leg): {} USDT", config.auto_pay_daily_cap as f64 / 1_000_000.0); + println!("Circuit breaker: >= {:.0}% of daily cap within {} min\n", + config.circuit_breaker_fraction * 100.0, config.circuit_breaker_window_minutes); + + let db = BridgeDatabase::load(&config.db_path)?; + let now = Utc::now(); + + for (label, leg) in [("Deposits (wUSDT payout)", LEG_DEPOSIT_WUSDT_PAYOUT), ("Withdrawals (USDT payout)", LEG_WITHDRAWAL_USDT_PAYOUT)] { + let vol_24h = db.recent_auto_payout_volume(leg, now, 24 * 60); + println!("{label}:"); + println!(" Rolling 24h auto-paid: {} / {} USDT", vol_24h as f64 / 1_000_000.0, config.auto_pay_daily_cap as f64 / 1_000_000.0); + } + + println!(); + if db.circuit_breaker_tripped { + println!("🚨 CIRCUIT BREAKER: TRIPPED at {}", db.circuit_breaker_tripped_at.as_deref().unwrap_or("unknown")); + println!(" Reason: {}", db.circuit_breaker_reason.as_deref().unwrap_or("none recorded")); + println!(" Run `reset-circuit-breaker` after human review to resume auto-pay."); + } else { + println!("✅ Circuit breaker: clear"); + } + + if std::env::var("AUTOMATION_KEY_MNEMONIC").is_err() { + println!("\n⚠️ AUTOMATION_KEY_MNEMONIC not set in this environment - auto-pay is fully disabled, everything falls back to manual 3-of-5 review."); + } + + Ok(()) +} + +fn reset_circuit_breaker(config: &BridgeConfig) -> Result<()> { + let mut db = BridgeDatabase::load(&config.db_path)?; + if !db.circuit_breaker_tripped { + println!("Circuit breaker was not tripped - nothing to do."); + return Ok(()); + } + println!("Clearing circuit breaker (was: {})", db.circuit_breaker_reason.as_deref().unwrap_or("no reason recorded")); + db.circuit_breaker_tripped = false; + db.circuit_breaker_reason = None; + db.circuit_breaker_tripped_at = None; + db.save(&config.db_path)?; + println!("Circuit breaker cleared - auto-pay will resume on the next eligible event."); Ok(()) } @@ -1040,10 +1555,12 @@ async fn main() -> Result<()> { show_addresses(&seed)?; } Commands::ListenDeposits => { - listen_deposits(&config).await?; + let automation = init_automation_context(&config).await; + listen_deposits(&config, &automation).await?; } Commands::ListenWithdrawals => { - listen_withdrawals(&config).await?; + let automation = init_automation_context(&config).await; + listen_withdrawals(&config, &automation).await?; } Commands::ListenAll => { listen_all(&config).await?; @@ -1063,6 +1580,12 @@ async fn main() -> Result<()> { Commands::Balances => { show_balances(&config).await?; } + Commands::AutomationStatus => { + show_automation_status(&config).await?; + } + Commands::ResetCircuitBreaker => { + reset_circuit_breaker(&config)?; + } } Ok(())