/** * SL Verification System - Post-Open Safety Checks * * Purpose: Verify stop loss orders are actually placed on-chain after position opens * Prevents Bug #76 (Silent SL Placement Failure) from leaving positions unprotected * * Architecture: Event-driven verification with exponential backoff (30s, 60s, 90s) * - Queries Drift on-chain state, not just database * - 3 verification attempts with increasing delays * - If all fail: Halt trading + close position immediately * * Rate Limit Impact: ~3-9 queries per position (vs 360/hour with interval-based) * * Created: Dec 16, 2025 (Bug #76 root cause + user mandate "better safe than sorry") */ import { getDriftService } from '../drift/client' import { getMarketConfig } from '../../config/trading' import { closePosition } from '../drift/orders' import { updateTradeState } from '../database/trades' import { sendTelegramMessage } from '../notifications/telegram' import { OrderType, Order } from '@drift-labs/sdk' // Global trading halt flag let tradingHalted = false let haltReason = '' export function isTradingHalted(): boolean { return tradingHalted } export function getHaltReason(): string { return haltReason } export function resetTradingHalt(): void { tradingHalted = false haltReason = '' console.log('✅ Trading halt reset - system re-enabled') } /** * Query Drift on-chain state to verify SL orders exist * Returns true if at least one SL order found (TRIGGER_MARKET or TRIGGER_LIMIT) */ export async function querySLOrdersFromDrift( symbol: string, marketIndex: number ): Promise<{ exists: boolean; orderCount: number; orderTypes: string[] }> { try { const driftService = getDriftService() const driftClient = driftService.getClient() // Get open orders from the drift client const allOrders = driftClient.getUser().getOpenOrders() const marketOrders = allOrders.filter( (order: Order) => order.marketIndex === marketIndex && order.reduceOnly === true ) // Find SL orders (TRIGGER_MARKET or TRIGGER_LIMIT) const slOrders = marketOrders.filter( (order: Order) => order.orderType === OrderType.TRIGGER_MARKET || order.orderType === OrderType.TRIGGER_LIMIT ) const orderTypes = slOrders.map((o: Order) => { if (o.orderType === OrderType.TRIGGER_MARKET) return 'TRIGGER_MARKET' if (o.orderType === OrderType.TRIGGER_LIMIT) return 'TRIGGER_LIMIT' return 'UNKNOWN' }) console.log(`🔍 SL Verification for ${symbol}:`) console.log(` Total reduce-only orders: ${marketOrders.length}`) console.log(` SL orders found: ${slOrders.length}`) if (slOrders.length > 0) { console.log(` Order types: ${orderTypes.join(', ')}`) } return { exists: slOrders.length > 0, orderCount: slOrders.length, orderTypes, } } catch (error) { console.error(`❌ Error querying SL orders from Drift:`, error) // On error, assume SL might exist (fail-open for transient failures) return { exists: true, orderCount: 0, orderTypes: [] } } } /** * Halt trading and close position immediately * Called when SL verification fails after all retries */ async function haltTradingAndClosePosition( tradeId: string, symbol: string, reason: string ): Promise { try { // Set global halt flag tradingHalted = true haltReason = reason console.error(`🚨🚨🚨 TRADING HALTED 🚨🚨🚨`) console.error(` Reason: ${reason}`) console.error(` Trade ID: ${tradeId}`) console.error(` Symbol: ${symbol}`) console.error(` Action: Closing position immediately for safety`) // Send critical Telegram alert await sendTelegramMessage(`🚨🚨🚨 CRITICAL: TRADING HALTED 🚨🚨🚨 Reason: ${reason} Trade ID: ${tradeId} Symbol: ${symbol} Action Taken: ✅ Closing position immediately (safety) ⛔ New trades blocked until manual reset Position left unprotected - closing to prevent losses. Manual reset required: Check logs and reset via API or Telegram.`) // Close position immediately console.log(`🔒 Closing ${symbol} position for safety...`) const closeResult = await closePosition({ symbol, percentToClose: 100, slippageTolerance: 0.02, // 2% slippage tolerance for emergency closes }) if (closeResult.success) { console.log(`✅ Position closed successfully: ${closeResult.transactionSignature}`) console.log(` Emergency closure reason: ${reason}`) // Update database with emergency exit reason const { getPrismaClient } = await import('../database/trades') const prisma = getPrismaClient() await prisma.trade.update({ where: { id: tradeId }, data: { exitReason: 'emergency', } }) } else { console.error(`❌ Failed to close position: ${closeResult.error}`) await sendTelegramMessage(`❌ CRITICAL: Failed to close position ${symbol} Close Error: ${closeResult.error} MANUAL INTERVENTION REQUIRED IMMEDIATELY`) } } catch (error) { console.error(`❌ Error in haltTradingAndClosePosition:`, error) await sendTelegramMessage(`❌ CRITICAL: Error halting trading and closing position Error: ${error instanceof Error ? error.message : String(error)} MANUAL INTERVENTION REQUIRED IMMEDIATELY`) } } /** * Verify SL orders exist with exponential backoff (30s, 60s, 90s) * If all 3 attempts fail: Halt trading + close position * * Usage: Call after position opened successfully * Example: await verifySLWithRetries(tradeId, symbol, marketIndex) */ export async function verifySLWithRetries( tradeId: string, symbol: string, marketIndex: number ): Promise { const delays = [30000, 60000, 90000] // 30s, 60s, 90s const maxAttempts = 3 console.log(`🛡️ Starting SL verification for ${symbol} (Trade: ${tradeId})`) console.log(` Verification schedule: 30s, 60s, 90s (3 attempts)`) for (let attempt = 1; attempt <= maxAttempts; attempt++) { const delay = delays[attempt - 1] console.log(`⏱️ Verification attempt ${attempt}/${maxAttempts} - waiting ${delay/1000}s...`) // Wait for scheduled delay await new Promise(resolve => setTimeout(resolve, delay)) // Query Drift on-chain state const slStatus = await querySLOrdersFromDrift(symbol, marketIndex) if (slStatus.exists) { console.log(`✅ SL VERIFIED on attempt ${attempt}/${maxAttempts}`) console.log(` Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`) console.log(` Verification timing: ${delay/1000}s after position open`) // Success - verification details logged above return // Success - exit retry loop } console.warn(`⚠️ SL NOT FOUND on attempt ${attempt}/${maxAttempts}`) console.warn(` Reduce-only orders: ${slStatus.orderCount}`) if (attempt < maxAttempts) { console.log(` Retrying in ${delays[attempt]/1000}s...`) } else { // All 3 attempts failed - CRITICAL FAILURE console.error(`❌ SL VERIFICATION FAILED after ${maxAttempts} attempts`) console.error(` Position is UNPROTECTED - initiating emergency procedures`) // Halt trading + close position await haltTradingAndClosePosition( tradeId, symbol, `SL verification failed after ${maxAttempts} attempts (30s, 60s, 90s). Position left unprotected - Bug #76 detected.` ) } } } /** * Check if trading is halted before accepting new trades * Returns { allow: boolean, reason: string } */ export function checkTradingAllowed(): { allow: boolean; reason: string } { if (tradingHalted) { return { allow: false, reason: `Trading halted: ${haltReason}. Manual reset required.`, } } return { allow: true, reason: '' } }