mirror of
https://gitlab.com/pezkuwichain/pezkuwi-sdk.git
synced 2026-10-20 18:56:23 +00:00
Add bounded automation-key auto-pay: per-tx cap, daily velocity cap, circuit breaker
Small/mid-size swaps no longer have to wait on 3-of-5 for every single transaction. usdt-bridge can now execute Assets.transfer_approved directly via the automation key's existing delegated allowance when a deposit/withdrawal passes all three independent guards: - max_single_tx (50,000 USDT): any single transaction above this always queues for manual multisig review, no exceptions. - auto_pay_daily_cap (200,000 USDT): a rolling 24h budget tracked separately per leg (wUSDT payouts vs USDT payouts draw on separate on-chain allowances), not a per-tx ceiling - lets many legitimate sub-cap swaps clear same-day. - circuit breaker: if >=50% of the daily cap is consumed within 30 minutes, auto-pay halts entirely until a human runs reset-circuit-breaker - a burst that fast reads as a compromised key or a bug, not organic demand. The automation key can only ever move funds via its pre-approved allowance (never mint/burn/set_team/transfer_ownership - those stay multisig-only), so its worst case is bounded by whatever allowance + daily cap remain, not unbounded custody. Withdrawal auto-pay releases real USDT immediately but defers the corresponding wUSDT burn (Owner/Admin-only, no delegation exists for it) to a periodic manual multisig batch - tracked via a distinct auto_paid_burn_pending status so it can't be silently forgotten. Auto-pay is opt-in at the infra level: AUTOMATION_KEY_MNEMONIC unset means 100% manual behavior, unchanged from before. Bumped pezbridge-bot's renewal_threshold/topup_amount so the next 3-of-5-approved allowance renewal scales to the new daily cap.
This commit is contained in:
Satoshi Qazi Muhammed committed
1 parent
a1555ce952
commit
96840fd17e
4 files changed
+661
-70
No files matched your search
@@ -12,6 +12,14 @@
|
||||
"custody_multisig_pezkuwi": "5GvwxmCDp3PC33KHoeWSgj3S7ocE7nzk1jiCCZMPSDBFeNcj",
|
||||
"serok_chat_id": 5362991649,
|
||||
"ops_chat_id": -5420200114,
|
||||
"max_single_tx": 50000000000,
|
||||
"auto_pay_daily_cap": 200000000000,
|
||||
"circuit_breaker_window_minutes": 30,
|
||||
"circuit_breaker_fraction": 0.5,
|
||||
"_automation_2026-07-14": {
|
||||
"note": "Small/mid swaps auto-pay via the automation key's Assets.transfer_approved allowance - no per-transaction 3-of-5 wait. max_single_tx (50,000 USDT) caps any single auto-paid transaction; auto_pay_daily_cap (200,000 USDT) is a rolling 24h budget tracked separately per leg (deposit wUSDT payouts vs withdrawal USDT payouts), NOT a per-tx ceiling - many sub-50K swaps can clear same-day. circuit_breaker_fraction/circuit_breaker_window_minutes halt all auto-pay if the daily budget is consumed abnormally fast (>=50% within 30 min) until a human runs `reset-circuit-breaker`. Anything over max_single_tx, or that would exceed the remaining daily budget, or while the breaker is tripped, ALWAYS falls back to full manual 3-of-5 review via pwap-web - this never bypasses the multisig, it only decides whether a given event needs it. Requires AUTOMATION_KEY_MNEMONIC env var to be set (see res/bridge_automation_key.json) - if unset, behavior is 100% manual, unchanged from before this feature.",
|
||||
"requires_on_chain_action": "The automation key's actual Assets.approve_transfer allowance is renewed by PezbridgeBot's 3-of-5-signed renewal flow (pezbridge_bot_config.json's renewal_threshold/topup_amount) - those were bumped alongside this to top up to a size that supports the new daily cap. The NEXT renewal signers approve via pwap-web will lift the real on-chain ceiling; until then, auto-pay is bounded by whatever allowance already exists on-chain today, which is smaller. This is a fail-safe direction: if the on-chain allowance runs out, the system simply falls back to manual, never fails open."
|
||||
},
|
||||
"_custody_migration_2026-07-13": {
|
||||
"note": "Custody is no longer a single hot key. wUSDT (asset 1000) owner/issuer/admin/freezer on Pezkuwi Asset Hub is now the 3-of-5 multisig below - seed_path/BRIDGE_SEED_MNEMONIC in this file is stale and must not be used to mint/burn/transfer_ownership going forward. listen_deposits/listen_withdrawals in src/main.rs were updated the same day: they now only detect events and record status=pending_multisig_approval - actual mint/burn/release requires 3 of the 5 signatories to approve via pwap-web's multisig UI (shared/lib/multisig.ts, shared/lib/usdt.ts).",
|
||||
"multisig_threshold": 3,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
[Unit]
|
||||
Description=usdt-bridge listen-all - USDT/wUSDT listener; auto-pays sub-cap swaps via the bounded automation-key allowance, everything else queues for 3-of-5
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=usdtbridge
|
||||
Group=usdtbridge
|
||||
WorkingDirectory=/opt/usdtbridge/data
|
||||
ExecStart=/opt/usdtbridge/bin/usdt-bridge --config /opt/usdtbridge/bin/bridge_config.json listen-all
|
||||
Environment=RUST_LOG=info
|
||||
EnvironmentFile=/etc/pezbridgebot/telegram_token.env
|
||||
EnvironmentFile=-/etc/usdtbridge/automation_key.env
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
# --- Hardening ---
|
||||
# Deployed 2026-07-14 on NEW-10 (89.116.25.180), same UTIL bucket as pezbridge-bot (no
|
||||
# consensus/fund-critical services here - see dks-ci-runners/vps2-dev-decommission-status
|
||||
# memory for the placement convention). UPDATED 2026-07-14: this process now also loads the
|
||||
# automation key's mnemonic (from a separate, tighter-scoped EnvironmentFile than the Telegram
|
||||
# token) and can sign+submit Assets.transfer_approved on both chains - but ONLY within the
|
||||
# multisig's pre-approved delegated allowance, capped per-tx (max_single_tx) and per rolling 24h
|
||||
# window (auto_pay_daily_cap) with a circuit breaker on burst consumption. It can never mint,
|
||||
# burn, set_team, or transfer_ownership - those stay exclusively multisig (3-of-5), same as
|
||||
# before. Worst case of this key being compromised is bounded by whatever allowance + daily cap
|
||||
# remain at the time, not unbounded custody - a materially different (and much smaller) risk
|
||||
# tier than the legacy single-key bot this whole migration replaced. Hardening kept as-is; the
|
||||
# automation_key.env file itself is root:usdtbridge 0640, separate from the Telegram token file
|
||||
# so a Telegram-token leak alone can never expose the signing key. The leading "-" on that
|
||||
# EnvironmentFile line is required (not decorative): the file does not exist until an operator
|
||||
# deliberately provisions it, and the unit must still start cleanly without it, matching the
|
||||
# code's own AUTOMATION_KEY_MNEMONIC-unset fallback (100% manual, auto-pay simply inactive).
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
PrivateDevices=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectKernelLogs=yes
|
||||
ProtectControlGroups=yes
|
||||
ProtectClock=yes
|
||||
ProtectHostname=yes
|
||||
RestrictSUIDSGID=yes
|
||||
RemoveIPC=yes
|
||||
LockPersonality=yes
|
||||
RestrictRealtime=yes
|
||||
RestrictNamespaces=yes
|
||||
MemoryDenyWriteExecute=yes
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallArchitectures=native
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
ReadWritePaths=/opt/usdtbridge/data
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -3,7 +3,7 @@
|
||||
"serok_chat_id": 5362991649,
|
||||
"ops_chat_id": -5420200114,
|
||||
"poll_interval_secs": 600,
|
||||
"renewal_threshold": 3000000000,
|
||||
"topup_amount": 10000000000,
|
||||
"renewal_threshold": 40000000000,
|
||||
"topup_amount": 200000000000,
|
||||
"state_path": "/opt/pezbridgebot/data/pezbridge_bot_state.json"
|
||||
}
|
||||
+591
-68
@@ -27,6 +27,7 @@
|
||||
//! - Fees: Configurable (default 0.1%)
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use chrono::{DateTime, Utc};
|
||||
use clap::{Parser, Subcommand};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sp_core::{crypto::Ss58Codec, sr25519, Pair};
|
||||
@@ -34,9 +35,29 @@ use std::path::PathBuf;
|
||||
use std::sync::LazyLock;
|
||||
use subxt::{OnlineClient, SubstrateConfig};
|
||||
use subxt::dynamic::{At, Value};
|
||||
use subxt_signer::bip39::Mnemonic;
|
||||
use subxt_signer::sr25519::Keypair as PolkadotKeypair;
|
||||
// Pezkuwi-side signing MUST go through this fork, not vanilla subxt - vanilla subxt's default
|
||||
// extrinsic params don't match Pezkuwi's signed-extension/hasher setup and produce "bad
|
||||
// signature" errors on submission (see examples/migrate_wusdt_to_multisig.rs, which hit this
|
||||
// first). Aliased to avoid clashing with the vanilla `Value`/`OnlineClient` already imported
|
||||
// above and used throughout this file for read-only queries on both chains.
|
||||
use pezkuwi_subxt::dynamic::Value as PValue;
|
||||
use pezkuwi_subxt::{OnlineClient as PezkuwiSigningClient, PezkuwiConfig};
|
||||
use pezkuwi_subxt_signer::bip39::Mnemonic as PezkuwiMnemonic;
|
||||
use pezkuwi_subxt_signer::sr25519::Keypair as PezkuwiKeypair;
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
use tracing::{info, warn, error};
|
||||
|
||||
// The automation key is a deliberately limited, server-held delegate: it can only ever move
|
||||
// funds the multisig has explicitly pre-approved via `Assets.approve_transfer`, up to whatever
|
||||
// allowance remains, and only via `transfer_approved` (never mint/burn/set_team/transfer_ownership
|
||||
// - those stay Owner/Admin-only, i.e. multisig-only, forever). This is categorically different
|
||||
// from the 5 signatories' own seeds - see the standing rule in this session: never load a
|
||||
// signatory's seed programmatically to bypass human multisig approval. This key exists
|
||||
// specifically so it CAN be loaded programmatically; that is its whole purpose.
|
||||
const AUTOMATION_KEY_ADDRESS: &str = "5GQu4PFUb1f3MTJ7i7c1CtLgDk3TVvpSW1VbQCRmfkMoC8cM";
|
||||
|
||||
// ============================================================================
|
||||
// Configuration
|
||||
// ============================================================================
|
||||
@@ -62,11 +83,27 @@ pub struct BridgeConfig {
|
||||
pub seed_path: PathBuf,
|
||||
/// Database path
|
||||
pub db_path: PathBuf,
|
||||
/// Maximum amount processed automatically in a single deposit/withdrawal (base units).
|
||||
/// Anything above this is left unprocessed for manual review - caps the blast radius
|
||||
/// of a single bad event (bug, reorg, compromised sender) instead of relaying it blindly.
|
||||
/// Per-transaction ceiling the automation key may execute on its own (base units). Anything
|
||||
/// above this ALWAYS queues for manual 3-of-5 review, regardless of remaining daily budget -
|
||||
/// caps the blast radius of a single bad event (bug, reorg, compromised sender, compromised
|
||||
/// automation key) to one transaction's worth, no matter how much daily headroom exists.
|
||||
#[serde(default = "default_max_single_tx")]
|
||||
pub max_single_tx: u128,
|
||||
/// Rolling 24h ceiling on total value the automation key may auto-pay out, tracked
|
||||
/// independently per leg (deposit payouts in wUSDT vs withdrawal payouts in USDT draw on
|
||||
/// separate on-chain allowances on separate chains). Bounds worst-case exposure from a
|
||||
/// compromised automation key to one day's volume rather than an unbounded amount, while
|
||||
/// still letting many legitimate sub-cap swaps clear same-day without human involvement.
|
||||
#[serde(default = "default_auto_pay_daily_cap")]
|
||||
pub auto_pay_daily_cap: u128,
|
||||
/// Circuit-breaker window: if `circuit_breaker_fraction` of the daily cap is consumed within
|
||||
/// this many minutes, auto-pay halts entirely (falls back to manual review) until a human
|
||||
/// clears it via the `reset-circuit-breaker` command - a burst that fast is far more
|
||||
/// consistent with a compromised key or a bug than organic demand.
|
||||
#[serde(default = "default_circuit_breaker_window_minutes")]
|
||||
pub circuit_breaker_window_minutes: i64,
|
||||
#[serde(default = "default_circuit_breaker_fraction")]
|
||||
pub circuit_breaker_fraction: f64,
|
||||
/// The 3-of-5 multisig custody account, Polkadot Asset Hub SS58 encoding. Deposits are
|
||||
/// watched for transfers TO this address - it replaced the old single-key custody address.
|
||||
#[serde(default = "default_multisig_polkadot")]
|
||||
@@ -85,7 +122,7 @@ pub struct BridgeConfig {
|
||||
}
|
||||
|
||||
fn default_max_single_tx() -> u128 {
|
||||
5_000_000_000 // 5,000 USDT (6 decimals) - conservative pilot-stage cap
|
||||
50_000_000_000 // 50,000 USDT (6 decimals) - small/mid-size swaps clear without waiting on 3-of-5
|
||||
}
|
||||
|
||||
fn default_multisig_polkadot() -> String {
|
||||
@@ -96,6 +133,18 @@ fn default_multisig_pezkuwi() -> String {
|
||||
"5GvwxmCDp3PC33KHoeWSgj3S7ocE7nzk1jiCCZMPSDBFeNcj".to_string()
|
||||
}
|
||||
|
||||
fn default_auto_pay_daily_cap() -> u128 {
|
||||
200_000_000_000 // 200,000 USDT/day - rolling 24h automation budget, per leg
|
||||
}
|
||||
|
||||
fn default_circuit_breaker_window_minutes() -> i64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_circuit_breaker_fraction() -> f64 {
|
||||
0.5 // >=50% of the daily cap consumed within the window trips the breaker
|
||||
}
|
||||
|
||||
impl Default for BridgeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
@@ -113,6 +162,9 @@ impl Default for BridgeConfig {
|
||||
custody_multisig_pezkuwi: default_multisig_pezkuwi(),
|
||||
serok_chat_id: 0,
|
||||
ops_chat_id: 0,
|
||||
auto_pay_daily_cap: default_auto_pay_daily_cap(),
|
||||
circuit_breaker_window_minutes: default_circuit_breaker_window_minutes(),
|
||||
circuit_breaker_fraction: default_circuit_breaker_fraction(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -172,6 +224,13 @@ enum Commands {
|
||||
|
||||
/// Check balances on both chains
|
||||
Balances,
|
||||
|
||||
/// Show automation-key auto-pay status: rolling 24h volume per leg vs caps, and whether
|
||||
/// the circuit breaker is tripped
|
||||
AutomationStatus,
|
||||
|
||||
/// Clear a tripped circuit breaker after human review - required before auto-pay resumes
|
||||
ResetCircuitBreaker,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -279,6 +338,29 @@ struct BridgeDatabase {
|
||||
deposits: Vec<DepositRecord>,
|
||||
withdrawals: Vec<WithdrawalRecord>,
|
||||
stats: BridgeStats,
|
||||
/// Ledger of automation-key auto-payouts, used to compute rolling-window volume per leg.
|
||||
/// Pruned to the last 48h on every write - only ever needs a 24h lookback plus slack.
|
||||
#[serde(default)]
|
||||
auto_payouts: Vec<AutoPayoutRecord>,
|
||||
#[serde(default)]
|
||||
circuit_breaker_tripped: bool,
|
||||
#[serde(default)]
|
||||
circuit_breaker_reason: Option<String>,
|
||||
#[serde(default)]
|
||||
circuit_breaker_tripped_at: Option<String>,
|
||||
}
|
||||
|
||||
/// One leg's identity for rolling-volume tracking - deposit payouts (wUSDT, Pezkuwi side) and
|
||||
/// withdrawal payouts (USDT, Polkadot side) draw on separate on-chain allowances and must never
|
||||
/// share a budget.
|
||||
const LEG_DEPOSIT_WUSDT_PAYOUT: &str = "deposit_wusdt_payout";
|
||||
const LEG_WITHDRAWAL_USDT_PAYOUT: &str = "withdrawal_usdt_payout";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct AutoPayoutRecord {
|
||||
leg: String,
|
||||
amount: u128,
|
||||
at: String, // RFC3339
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -375,6 +457,37 @@ impl BridgeDatabase {
|
||||
fn is_withdrawal_processed(&self, pezkuwi_tx_hash: &str) -> bool {
|
||||
self.withdrawals.iter().any(|w| w.pezkuwi_tx_hash == pezkuwi_tx_hash)
|
||||
}
|
||||
|
||||
/// Sum of auto-payouts on `leg` within the last `window_minutes` minutes, relative to `now`.
|
||||
/// Used for both the 24h daily-cap check (window_minutes = 1440) and the fast-burst
|
||||
/// circuit-breaker check (window_minutes = config.circuit_breaker_window_minutes).
|
||||
fn recent_auto_payout_volume(&self, leg: &str, now: DateTime<Utc>, window_minutes: i64) -> u128 {
|
||||
self.auto_payouts
|
||||
.iter()
|
||||
.filter(|p| p.leg == leg)
|
||||
.filter_map(|p| DateTime::parse_from_rfc3339(&p.at).ok().map(|t| (t.with_timezone(&Utc), p.amount)))
|
||||
.filter(|(t, _)| now.signed_duration_since(*t) < chrono::Duration::minutes(window_minutes))
|
||||
.map(|(_, a)| a)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Records a completed auto-payout and prunes entries older than 48h (only a 24h lookback is
|
||||
/// ever needed - the extra slack just avoids pruning right at the boundary of a live check).
|
||||
fn record_auto_payout(&mut self, leg: &str, amount: u128, now: DateTime<Utc>) {
|
||||
self.auto_payouts.push(AutoPayoutRecord { leg: leg.to_string(), amount, at: now.to_rfc3339() });
|
||||
let cutoff = now - chrono::Duration::hours(48);
|
||||
self.auto_payouts.retain(|p| {
|
||||
DateTime::parse_from_rfc3339(&p.at)
|
||||
.map(|t| t.with_timezone(&Utc) > cutoff)
|
||||
.unwrap_or(true) // keep anything unparseable rather than silently drop history
|
||||
});
|
||||
}
|
||||
|
||||
fn trip_circuit_breaker(&mut self, reason: String, now: DateTime<Utc>) {
|
||||
self.circuit_breaker_tripped = true;
|
||||
self.circuit_breaker_reason = Some(reason);
|
||||
self.circuit_breaker_tripped_at = Some(now.to_rfc3339());
|
||||
}
|
||||
}
|
||||
|
||||
/// Guards every load-mutate-save critical section on `BridgeDatabase`. `listen_deposits` and
|
||||
@@ -509,6 +622,213 @@ async fn notify_telegram(config: &BridgeConfig, msg: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Automation key: bounded auto-pay via Assets.transfer_approved
|
||||
//
|
||||
// The automation key holds ONLY a delegated `Assets.approve_transfer` allowance from the
|
||||
// multisig - it can never mint, burn, set_team, or transfer_ownership. Its worst case (private
|
||||
// key stolen) is bounded by whatever allowance remains on-chain at the time, which in turn is
|
||||
// bounded by `max_single_tx` per transaction and kept small by design (renewed in modest
|
||||
// top-ups, requiring fresh 3-of-5 approval each time - see pezbridge_bot.rs). This module adds
|
||||
// two more independent layers on top of that on-chain bound: a rolling 24h volume cap per leg,
|
||||
// and a burst-rate circuit breaker that halts all auto-pay if that cap is consumed too fast.
|
||||
// ============================================================================
|
||||
|
||||
/// Both chain-specific signing keypairs for the SAME automation key (same seed, same sr25519
|
||||
/// key - just wrapped in the two different subxt fork types each chain's extrinsic params need).
|
||||
/// `None` fields mean the corresponding on-chain client failed to connect at startup; auto-pay
|
||||
/// on that leg is then simply skipped (falls back to today's fully-manual pending-record path).
|
||||
struct AutomationContext {
|
||||
pezkuwi_signing_client: Option<PezkuwiSigningClient<PezkuwiConfig>>,
|
||||
pezkuwi_keypair: Option<PezkuwiKeypair>,
|
||||
polkadot_signing_client: Option<OnlineClient<SubstrateConfig>>,
|
||||
polkadot_keypair: Option<PolkadotKeypair>,
|
||||
}
|
||||
|
||||
impl AutomationContext {
|
||||
fn disabled() -> Self {
|
||||
Self { pezkuwi_signing_client: None, pezkuwi_keypair: None, polkadot_signing_client: None, polkadot_keypair: None }
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the automation key's mnemonic from `AUTOMATION_KEY_MNEMONIC` and connects signing
|
||||
/// clients on both chains. If the env var isn't set, returns a fully-disabled context - the
|
||||
/// system behaves exactly as it did before this feature existed (100% manual multisig review).
|
||||
/// This is deliberate: auto-pay must never be "on by default" just because the binary was
|
||||
/// rebuilt: it activates only once an operator explicitly provisions the credential.
|
||||
async fn init_automation_context(config: &BridgeConfig) -> AutomationContext {
|
||||
let Ok(mnemonic_str) = std::env::var("AUTOMATION_KEY_MNEMONIC") else {
|
||||
warn!("AUTOMATION_KEY_MNEMONIC not set - auto-pay disabled, all deposits/withdrawals will queue for manual 3-of-5 review (this is safe, just fully manual)");
|
||||
return AutomationContext::disabled();
|
||||
};
|
||||
|
||||
let pezkuwi_keypair = (|| -> Result<PezkuwiKeypair> {
|
||||
let m = PezkuwiMnemonic::parse(&mnemonic_str).context("parsing automation mnemonic (pezkuwi)")?;
|
||||
let kp = PezkuwiKeypair::from_phrase(&m, None).context("deriving pezkuwi keypair")?;
|
||||
let addr = kp.public_key().to_account_id().to_string();
|
||||
if addr != AUTOMATION_KEY_ADDRESS {
|
||||
return Err(anyhow!("AUTOMATION_KEY_MNEMONIC derives {addr}, expected {AUTOMATION_KEY_ADDRESS} - refusing to use a mismatched key"));
|
||||
}
|
||||
Ok(kp)
|
||||
})();
|
||||
|
||||
let polkadot_keypair = (|| -> Result<PolkadotKeypair> {
|
||||
let m = Mnemonic::parse(&mnemonic_str).context("parsing automation mnemonic (polkadot)")?;
|
||||
PolkadotKeypair::from_phrase(&m, None).context("deriving polkadot keypair")
|
||||
})();
|
||||
|
||||
let (pezkuwi_keypair, polkadot_keypair) = match (pezkuwi_keypair, polkadot_keypair) {
|
||||
(Ok(p), Ok(d)) => (Some(p), Some(d)),
|
||||
(Err(e), _) | (_, Err(e)) => {
|
||||
error!("Failed to derive automation keypair(s), auto-pay disabled: {e:#}");
|
||||
return AutomationContext::disabled();
|
||||
}
|
||||
};
|
||||
|
||||
let pezkuwi_signing_client = match PezkuwiSigningClient::<PezkuwiConfig>::from_insecure_url(&config.pezkuwi_rpc).await {
|
||||
Ok(c) => Some(c),
|
||||
Err(e) => { error!("Auto-pay (deposit/wUSDT leg) disabled - failed to connect signing client: {e:#}"); None }
|
||||
};
|
||||
let polkadot_signing_client = match OnlineClient::<SubstrateConfig>::from_url(&config.polkadot_rpc).await {
|
||||
Ok(c) => Some(c),
|
||||
Err(e) => { error!("Auto-pay (withdrawal/USDT leg) disabled - failed to connect signing client: {e:#}"); None }
|
||||
};
|
||||
|
||||
info!("Automation key loaded ({AUTOMATION_KEY_ADDRESS}) - auto-pay ENABLED where signing clients connected");
|
||||
AutomationContext { pezkuwi_signing_client, pezkuwi_keypair, polkadot_signing_client, polkadot_keypair }
|
||||
}
|
||||
|
||||
fn multi_address_id_pezkuwi(bytes: [u8; 32]) -> PValue {
|
||||
PValue::unnamed_variant("Id", [PValue::from_bytes(bytes)])
|
||||
}
|
||||
|
||||
fn multi_address_id_polkadot(bytes: [u8; 32]) -> Value {
|
||||
Value::unnamed_variant("Id", [Value::from_bytes(bytes)])
|
||||
}
|
||||
|
||||
/// Submits `Assets.transfer_approved(asset_id, owner, destination, amount)` on Pezkuwi Asset Hub,
|
||||
/// signed by the automation key, and waits for on-chain dispatch success. Used for the
|
||||
/// USDT->wUSDT deposit leg (paying wUSDT out of the multisig's seed pool).
|
||||
async fn execute_wusdt_auto_payout(
|
||||
client: &PezkuwiSigningClient<PezkuwiConfig>,
|
||||
keypair: &PezkuwiKeypair,
|
||||
asset_id: u32,
|
||||
owner_bytes: [u8; 32],
|
||||
destination_bytes: [u8; 32],
|
||||
amount: u128,
|
||||
) -> Result<String> {
|
||||
let call = pezkuwi_subxt::dynamic::tx(
|
||||
"Assets",
|
||||
"transfer_approved",
|
||||
vec![
|
||||
PValue::primitive(asset_id.into()),
|
||||
multi_address_id_pezkuwi(owner_bytes),
|
||||
multi_address_id_pezkuwi(destination_bytes),
|
||||
PValue::u128(amount),
|
||||
],
|
||||
);
|
||||
let mut progress = client.tx().sign_and_submit_then_watch_default(&call, keypair).await
|
||||
.context("submitting transfer_approved (pezkuwi)")?;
|
||||
let hash = format!("0x{}", hex::encode(progress.extrinsic_hash().as_ref()));
|
||||
use pezkuwi_subxt::tx::TxStatus;
|
||||
loop {
|
||||
match progress.next().await {
|
||||
Some(Ok(TxStatus::InBestBlock(details))) => {
|
||||
details.wait_for_success().await.map_err(|e| anyhow!("transfer_approved dispatch failed: {e}"))?;
|
||||
return Ok(hash);
|
||||
}
|
||||
Some(Ok(TxStatus::Error { message })) => return Err(anyhow!("tx error: {message}")),
|
||||
Some(Ok(TxStatus::Invalid { message })) => return Err(anyhow!("tx invalid: {message}")),
|
||||
Some(Ok(TxStatus::Dropped { message })) => return Err(anyhow!("tx dropped: {message}")),
|
||||
Some(Err(e)) => return Err(anyhow!("stream error: {e}")),
|
||||
None => return Err(anyhow!("stream ended without a result")),
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Same as `execute_wusdt_auto_payout` but on Polkadot Asset Hub via vanilla subxt (correct here
|
||||
/// - the bad-signature gotcha only applies to Pezkuwi's custom chain). Used for the wUSDT->USDT
|
||||
/// withdrawal leg (paying real USDT out of the multisig's Polkadot-side reserve).
|
||||
async fn execute_usdt_auto_payout(
|
||||
client: &OnlineClient<SubstrateConfig>,
|
||||
keypair: &PolkadotKeypair,
|
||||
asset_id: u32,
|
||||
owner_bytes: [u8; 32],
|
||||
destination_bytes: [u8; 32],
|
||||
amount: u128,
|
||||
) -> Result<String> {
|
||||
let call = subxt::dynamic::tx(
|
||||
"Assets",
|
||||
"transfer_approved",
|
||||
vec![
|
||||
Value::primitive(asset_id.into()),
|
||||
multi_address_id_polkadot(owner_bytes),
|
||||
multi_address_id_polkadot(destination_bytes),
|
||||
Value::u128(amount),
|
||||
],
|
||||
);
|
||||
let mut progress = client.tx().sign_and_submit_then_watch_default(&call, keypair).await
|
||||
.context("submitting transfer_approved (polkadot)")?;
|
||||
let hash = format!("0x{}", hex::encode(progress.extrinsic_hash().as_ref()));
|
||||
use subxt::tx::TxStatus;
|
||||
loop {
|
||||
match progress.next().await {
|
||||
Some(Ok(TxStatus::InBestBlock(details))) => {
|
||||
details.wait_for_success().await.map_err(|e| anyhow!("transfer_approved dispatch failed: {e}"))?;
|
||||
return Ok(hash);
|
||||
}
|
||||
Some(Ok(TxStatus::Error { message })) => return Err(anyhow!("tx error: {message}")),
|
||||
Some(Ok(TxStatus::Invalid { message })) => return Err(anyhow!("tx invalid: {message}")),
|
||||
Some(Ok(TxStatus::Dropped { message })) => return Err(anyhow!("tx dropped: {message}")),
|
||||
Some(Err(e)) => return Err(anyhow!("stream error: {e}")),
|
||||
None => return Err(anyhow!("stream ended without a result")),
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared eligibility gate for both legs: circuit breaker must be clear, amount within per-tx
|
||||
/// cap (redundant with the caller's existing max_single_tx check, checked again here so this
|
||||
/// function is safe to call from anywhere), and (existing + amount) must stay within the
|
||||
/// rolling 24h cap for this leg. Returns the disqualifying reason as Err(reason) for logging/
|
||||
/// alerting, or Ok(()) if auto-pay should proceed.
|
||||
fn check_auto_pay_eligible(db: &BridgeDatabase, config: &BridgeConfig, leg: &str, amount: u128, now: DateTime<Utc>) -> std::result::Result<(), String> {
|
||||
if db.circuit_breaker_tripped {
|
||||
return Err(format!("circuit breaker is tripped ({}) - auto-pay halted until a human runs reset-circuit-breaker", db.circuit_breaker_reason.as_deref().unwrap_or("no reason recorded")));
|
||||
}
|
||||
if amount > config.max_single_tx {
|
||||
return Err(format!("amount {} exceeds max_single_tx {}", amount, config.max_single_tx));
|
||||
}
|
||||
let existing = db.recent_auto_payout_volume(leg, now, 24 * 60);
|
||||
if existing + amount > config.auto_pay_daily_cap {
|
||||
return Err(format!(
|
||||
"would exceed daily auto-pay cap for {leg}: {} already used + {} requested > {} cap",
|
||||
existing, amount, config.auto_pay_daily_cap
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// After a successful auto-payout, checks whether the burst-rate circuit breaker should trip -
|
||||
/// i.e. whether `circuit_breaker_fraction` of the daily cap was consumed within
|
||||
/// `circuit_breaker_window_minutes`. Call this AFTER `record_auto_payout` so the just-completed
|
||||
/// payout is included in the window sum.
|
||||
fn maybe_trip_circuit_breaker(db: &mut BridgeDatabase, config: &BridgeConfig, leg: &str, now: DateTime<Utc>) -> Option<String> {
|
||||
let recent = db.recent_auto_payout_volume(leg, now, config.circuit_breaker_window_minutes);
|
||||
let threshold = (config.auto_pay_daily_cap as f64 * config.circuit_breaker_fraction) as u128;
|
||||
if recent >= threshold {
|
||||
let reason = format!(
|
||||
"{leg}: {} auto-paid within {} min (>= {:.0}% of daily cap {}) - burst rate far exceeds normal organic demand",
|
||||
recent, config.circuit_breaker_window_minutes, config.circuit_breaker_fraction * 100.0, config.auto_pay_daily_cap
|
||||
);
|
||||
db.trip_circuit_breaker(reason.clone(), now);
|
||||
Some(reason)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Bridge Operations
|
||||
// ============================================================================
|
||||
@@ -662,7 +982,7 @@ fn calculate_fee(amount: u128, fee_basis_points: u32) -> u128 {
|
||||
amount * fee_basis_points as u128 / 10_000
|
||||
}
|
||||
|
||||
async fn listen_deposits(config: &BridgeConfig) -> Result<()> {
|
||||
async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext) -> Result<()> {
|
||||
let polkadot_addr = &config.custody_multisig_polkadot;
|
||||
|
||||
println!("╔══════════════════════════════════════════════════════════════╗");
|
||||
@@ -721,10 +1041,8 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> {
|
||||
|
||||
if to_bytes.is_some() && to_bytes == target_bytes {
|
||||
{
|
||||
let from_str = from_field
|
||||
.and_then(account_id_from_dynamic)
|
||||
.map(|b| account_bytes_to_ss58(b, 0))
|
||||
.unwrap_or_default();
|
||||
let from_bytes_opt = from_field.and_then(account_id_from_dynamic);
|
||||
let from_str = from_bytes_opt.map(|b| account_bytes_to_ss58(b, 0)).unwrap_or_default();
|
||||
let tx_hash = format!("block-{}-{}", block_number, from_str);
|
||||
|
||||
info!("📥 DEPOSIT DETECTED!");
|
||||
@@ -756,32 +1074,99 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> {
|
||||
// correct rather than a placeholder.
|
||||
match convert_ss58_prefix(&from_str, 42) {
|
||||
Ok(pezkuwi_recipient) => {
|
||||
info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-minting.");
|
||||
info!(
|
||||
" 3 of 5 signatories must approve: Assets.transfer({}, {}, {}) from the multisig's seed pool",
|
||||
config.pezkuwi_wusdt_asset_id, pezkuwi_recipient, net_amount
|
||||
);
|
||||
info!(" (fee: {} USDT, net: {} USDT) - use pwap-web's multisig UI.",
|
||||
fee as f64 / 1_000_000.0, net_amount as f64 / 1_000_000.0);
|
||||
let now = Utc::now();
|
||||
let auto_pay_attempt = match (from_bytes_opt, &automation.pezkuwi_signing_client, &automation.pezkuwi_keypair) {
|
||||
(Some(dest_bytes), Some(client), Some(keypair)) => {
|
||||
match check_auto_pay_eligible(&db, config, LEG_DEPOSIT_WUSDT_PAYOUT, net_amount, now) {
|
||||
Ok(()) => {
|
||||
let owner_bytes = sp_core::crypto::AccountId32::from_ss58check(&config.custody_multisig_pezkuwi)
|
||||
.map(Into::<[u8; 32]>::into)
|
||||
.map_err(|e| format!("bad custody_multisig_pezkuwi config: {e:?}"));
|
||||
match owner_bytes {
|
||||
Ok(owner_bytes) => Some(execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await),
|
||||
Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None }
|
||||
}
|
||||
}
|
||||
Err(reason) => { info!(" ℹ️ Not auto-paying (falls back to manual): {reason}"); None }
|
||||
}
|
||||
}
|
||||
_ => None, // automation not configured/connected - always-manual fallback
|
||||
};
|
||||
|
||||
db.add_deposit(DepositRecord {
|
||||
id: db.next_deposit_id(),
|
||||
polkadot_tx_hash: tx_hash.clone(),
|
||||
polkadot_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
pezkuwi_tx_hash: None,
|
||||
status: "pending_multisig_approval".to_string(),
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
notify_telegram(config, &format!(
|
||||
"📥 New deposit pending approval: {} USDT from {} -> mint {} wUSDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient
|
||||
)).await;
|
||||
match auto_pay_attempt {
|
||||
Some(Ok(pezkuwi_tx_hash)) => {
|
||||
info!(" ✅ AUTO-PAID: {} wUSDT -> {} (tx {})", net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash);
|
||||
db.record_auto_payout(LEG_DEPOSIT_WUSDT_PAYOUT, net_amount, now);
|
||||
let breaker_tripped = maybe_trip_circuit_breaker(&mut db, config, LEG_DEPOSIT_WUSDT_PAYOUT, now);
|
||||
db.add_deposit(DepositRecord {
|
||||
id: db.next_deposit_id(),
|
||||
polkadot_tx_hash: tx_hash.clone(),
|
||||
polkadot_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
amount, fee, net_amount,
|
||||
pezkuwi_tx_hash: Some(pezkuwi_tx_hash.clone()),
|
||||
status: "auto_completed".to_string(),
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: Some(now.to_rfc3339()),
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
if let Some(reason) = breaker_tripped {
|
||||
notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (deposit/wUSDT leg): {reason}. All further deposits now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await;
|
||||
} else {
|
||||
notify_telegram(config, &format!(
|
||||
"✅ Auto-paid deposit: {} USDT from {} -> {} wUSDT (net of fee) to {}. tx {}",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash
|
||||
)).await;
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
error!(" ❌ Auto-pay execution FAILED, falling back to manual review: {e:#}");
|
||||
notify_telegram(config, &format!(
|
||||
"⚠️ Auto-pay attempt failed for deposit {} USDT from {} - falling back to manual 3-of-5 review. Error: {e:#}",
|
||||
amount as f64 / 1_000_000.0, from_str
|
||||
)).await;
|
||||
db.add_deposit(DepositRecord {
|
||||
id: db.next_deposit_id(),
|
||||
polkadot_tx_hash: tx_hash.clone(),
|
||||
polkadot_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
amount, fee, net_amount,
|
||||
pezkuwi_tx_hash: None,
|
||||
status: "pending_multisig_approval".to_string(),
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
}
|
||||
None => {
|
||||
info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-minting.");
|
||||
info!(
|
||||
" 3 of 5 signatories must approve: Assets.transfer({}, {}, {}) from the multisig's seed pool",
|
||||
config.pezkuwi_wusdt_asset_id, pezkuwi_recipient, net_amount
|
||||
);
|
||||
info!(" (fee: {} USDT, net: {} USDT) - use pwap-web's multisig UI.",
|
||||
fee as f64 / 1_000_000.0, net_amount as f64 / 1_000_000.0);
|
||||
|
||||
db.add_deposit(DepositRecord {
|
||||
id: db.next_deposit_id(),
|
||||
polkadot_tx_hash: tx_hash.clone(),
|
||||
polkadot_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
pezkuwi_tx_hash: None,
|
||||
status: "pending_multisig_approval".to_string(),
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
notify_telegram(config, &format!(
|
||||
"📥 New deposit pending approval: {} USDT from {} -> mint {} wUSDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient
|
||||
)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(" ❌ Could not derive Pezkuwi recipient from sender {}: {}", from_str, e);
|
||||
@@ -807,7 +1192,7 @@ async fn listen_deposits(config: &BridgeConfig) -> Result<()> {
|
||||
/// bridge was missing - a mismatch here means the peg is under-collateralized and must halt,
|
||||
/// not silently short-pay a user), then release on Polkadot. Mirrors listen_deposits() and
|
||||
/// shares its event-matching caveat - see the comment there.
|
||||
async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> {
|
||||
async fn listen_withdrawals(config: &BridgeConfig, automation: &AutomationContext) -> Result<()> {
|
||||
let polkadot_bridge_addr = &config.custody_multisig_polkadot;
|
||||
let pezkuwi_bridge_addr = &config.custody_multisig_pezkuwi;
|
||||
|
||||
@@ -857,10 +1242,8 @@ async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> {
|
||||
|
||||
if to_bytes.is_some() && to_bytes == target_bytes {
|
||||
{
|
||||
let from_str = from_field
|
||||
.and_then(account_id_from_dynamic)
|
||||
.map(|b| account_bytes_to_ss58(b, 42))
|
||||
.unwrap_or_default();
|
||||
let from_bytes_opt = from_field.and_then(account_id_from_dynamic);
|
||||
let from_str = from_bytes_opt.map(|b| account_bytes_to_ss58(b, 42)).unwrap_or_default();
|
||||
let tx_hash = format!("block-{}-{}", block_number, from_str);
|
||||
|
||||
info!("📤 WITHDRAWAL DETECTED!");
|
||||
@@ -929,33 +1312,114 @@ async fn listen_withdrawals(config: &BridgeConfig) -> Result<()> {
|
||||
|
||||
match convert_ss58_prefix(&from_str, 0) {
|
||||
Ok(polkadot_recipient) => {
|
||||
info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-burning/releasing.");
|
||||
info!(
|
||||
" 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot",
|
||||
amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount
|
||||
);
|
||||
info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
|
||||
reserve as f64 / 1_000_000.0);
|
||||
let now = Utc::now();
|
||||
// Never attempt auto-pay against an already-known reserve shortfall -
|
||||
// that's exactly the case that must always go to human review.
|
||||
let auto_pay_attempt = if reserve < net_amount {
|
||||
None
|
||||
} else {
|
||||
match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) {
|
||||
(Some(dest_bytes), Some(client), Some(keypair)) => {
|
||||
match check_auto_pay_eligible(&db, config, LEG_WITHDRAWAL_USDT_PAYOUT, net_amount, now) {
|
||||
Ok(()) => {
|
||||
let owner_bytes = sp_core::crypto::AccountId32::from_ss58check(polkadot_bridge_addr)
|
||||
.map(Into::<[u8; 32]>::into)
|
||||
.map_err(|e| format!("bad custody_multisig_polkadot config: {e:?}"));
|
||||
match owner_bytes {
|
||||
Ok(owner_bytes) => Some(execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await),
|
||||
Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None }
|
||||
}
|
||||
}
|
||||
Err(reason) => { info!(" ℹ️ Not auto-paying (falls back to manual): {reason}"); None }
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address: polkadot_recipient,
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
polkadot_tx_hash: None,
|
||||
status: if reserve < net_amount {
|
||||
"pending_multisig_approval_reserve_shortfall".to_string()
|
||||
} else {
|
||||
"pending_multisig_approval".to_string()
|
||||
},
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
match auto_pay_attempt {
|
||||
Some(Ok(polkadot_tx_hash)) => {
|
||||
// The wUSDT the user sent in is NOT burned here - burn is
|
||||
// Owner/Admin-only (multisig-only, no delegation exists for
|
||||
// it). Paying the user out is what's time-critical; the
|
||||
// matching burn is deferred bookkeeping the signatories batch
|
||||
// periodically via pwap-web - tracked explicitly by this
|
||||
// status so it can never be silently forgotten.
|
||||
info!(" ✅ AUTO-PAID: {} USDT -> {} (tx {}); wUSDT burn still pending (multisig-only)", net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash);
|
||||
db.record_auto_payout(LEG_WITHDRAWAL_USDT_PAYOUT, net_amount, now);
|
||||
let breaker_tripped = maybe_trip_circuit_breaker(&mut db, config, LEG_WITHDRAWAL_USDT_PAYOUT, now);
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address: polkadot_recipient.clone(),
|
||||
amount, fee, net_amount,
|
||||
polkadot_tx_hash: Some(polkadot_tx_hash.clone()),
|
||||
status: "auto_paid_burn_pending".to_string(),
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: Some(now.to_rfc3339()),
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
if let Some(reason) = breaker_tripped {
|
||||
notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (withdrawal/USDT leg): {reason}. All further withdrawals now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await;
|
||||
} else {
|
||||
notify_telegram(config, &format!(
|
||||
"✅ Auto-paid withdrawal: {} wUSDT from {} -> {} USDT (net of fee) to {}. tx {}. wUSDT burn still pending - batch via pwap-web when convenient.",
|
||||
amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash
|
||||
)).await;
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
error!(" ❌ Auto-pay execution FAILED, falling back to manual review: {e:#}");
|
||||
notify_telegram(config, &format!(
|
||||
"⚠️ Auto-pay attempt failed for withdrawal {} USDT from {} - falling back to manual 3-of-5 review. Error: {e:#}",
|
||||
amount as f64 / 1_000_000.0, from_str
|
||||
)).await;
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address: polkadot_recipient,
|
||||
amount, fee, net_amount,
|
||||
polkadot_tx_hash: None,
|
||||
status: "pending_multisig_approval".to_string(),
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
}
|
||||
None => {
|
||||
info!(" ⏳ AWAITING MULTISIG APPROVAL - not auto-burning/releasing.");
|
||||
info!(
|
||||
" 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot",
|
||||
amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount
|
||||
);
|
||||
info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
|
||||
reserve as f64 / 1_000_000.0);
|
||||
|
||||
db.add_withdrawal(WithdrawalRecord {
|
||||
id: db.next_withdrawal_id(),
|
||||
pezkuwi_tx_hash: tx_hash.clone(),
|
||||
pezkuwi_block: block_number as u64,
|
||||
sender_address: from_str.clone(),
|
||||
destination_address: polkadot_recipient,
|
||||
amount,
|
||||
fee,
|
||||
net_amount,
|
||||
polkadot_tx_hash: None,
|
||||
status: if reserve < net_amount {
|
||||
"pending_multisig_approval_reserve_shortfall".to_string()
|
||||
} else {
|
||||
"pending_multisig_approval".to_string()
|
||||
},
|
||||
created_at: now.to_rfc3339(),
|
||||
processed_at: None,
|
||||
});
|
||||
db.save(&config.db_path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(" ❌ Could not derive Polkadot recipient from sender {}: {}", from_str, e);
|
||||
@@ -1000,7 +1464,58 @@ async fn process_withdrawals(config: &BridgeConfig) -> Result<()> {
|
||||
}
|
||||
|
||||
async fn listen_all(config: &BridgeConfig) -> Result<()> {
|
||||
tokio::try_join!(listen_deposits(config), listen_withdrawals(config))?;
|
||||
let automation = init_automation_context(config).await;
|
||||
tokio::try_join!(listen_deposits(config, &automation), listen_withdrawals(config, &automation))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn show_automation_status(config: &BridgeConfig) -> Result<()> {
|
||||
println!("╔══════════════════════════════════════════════════════════════╗");
|
||||
println!("║ AUTOMATION KEY AUTO-PAY STATUS ║");
|
||||
println!("╚══════════════════════════════════════════════════════════════╝\n");
|
||||
|
||||
println!("Per-tx cap (max_single_tx): {} USDT", config.max_single_tx as f64 / 1_000_000.0);
|
||||
println!("Daily cap (per leg): {} USDT", config.auto_pay_daily_cap as f64 / 1_000_000.0);
|
||||
println!("Circuit breaker: >= {:.0}% of daily cap within {} min\n",
|
||||
config.circuit_breaker_fraction * 100.0, config.circuit_breaker_window_minutes);
|
||||
|
||||
let db = BridgeDatabase::load(&config.db_path)?;
|
||||
let now = Utc::now();
|
||||
|
||||
for (label, leg) in [("Deposits (wUSDT payout)", LEG_DEPOSIT_WUSDT_PAYOUT), ("Withdrawals (USDT payout)", LEG_WITHDRAWAL_USDT_PAYOUT)] {
|
||||
let vol_24h = db.recent_auto_payout_volume(leg, now, 24 * 60);
|
||||
println!("{label}:");
|
||||
println!(" Rolling 24h auto-paid: {} / {} USDT", vol_24h as f64 / 1_000_000.0, config.auto_pay_daily_cap as f64 / 1_000_000.0);
|
||||
}
|
||||
|
||||
println!();
|
||||
if db.circuit_breaker_tripped {
|
||||
println!("🚨 CIRCUIT BREAKER: TRIPPED at {}", db.circuit_breaker_tripped_at.as_deref().unwrap_or("unknown"));
|
||||
println!(" Reason: {}", db.circuit_breaker_reason.as_deref().unwrap_or("none recorded"));
|
||||
println!(" Run `reset-circuit-breaker` after human review to resume auto-pay.");
|
||||
} else {
|
||||
println!("✅ Circuit breaker: clear");
|
||||
}
|
||||
|
||||
if std::env::var("AUTOMATION_KEY_MNEMONIC").is_err() {
|
||||
println!("\n⚠️ AUTOMATION_KEY_MNEMONIC not set in this environment - auto-pay is fully disabled, everything falls back to manual 3-of-5 review.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset_circuit_breaker(config: &BridgeConfig) -> Result<()> {
|
||||
let mut db = BridgeDatabase::load(&config.db_path)?;
|
||||
if !db.circuit_breaker_tripped {
|
||||
println!("Circuit breaker was not tripped - nothing to do.");
|
||||
return Ok(());
|
||||
}
|
||||
println!("Clearing circuit breaker (was: {})", db.circuit_breaker_reason.as_deref().unwrap_or("no reason recorded"));
|
||||
db.circuit_breaker_tripped = false;
|
||||
db.circuit_breaker_reason = None;
|
||||
db.circuit_breaker_tripped_at = None;
|
||||
db.save(&config.db_path)?;
|
||||
println!("Circuit breaker cleared - auto-pay will resume on the next eligible event.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1040,10 +1555,12 @@ async fn main() -> Result<()> {
|
||||
show_addresses(&seed)?;
|
||||
}
|
||||
Commands::ListenDeposits => {
|
||||
listen_deposits(&config).await?;
|
||||
let automation = init_automation_context(&config).await;
|
||||
listen_deposits(&config, &automation).await?;
|
||||
}
|
||||
Commands::ListenWithdrawals => {
|
||||
listen_withdrawals(&config).await?;
|
||||
let automation = init_automation_context(&config).await;
|
||||
listen_withdrawals(&config, &automation).await?;
|
||||
}
|
||||
Commands::ListenAll => {
|
||||
listen_all(&config).await?;
|
||||
@@ -1063,6 +1580,12 @@ async fn main() -> Result<()> {
|
||||
Commands::Balances => {
|
||||
show_balances(&config).await?;
|
||||
}
|
||||
Commands::AutomationStatus => {
|
||||
show_automation_status(&config).await?;
|
||||
}
|
||||
Commands::ResetCircuitBreaker => {
|
||||
reset_circuit_breaker(&config)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in new issue
Block a user