/** * SL Verification System - Post-Open Safety Checks * * Purpose: Verify stop loss orders are actually placed on-chain after position opens * Prevents Bug #76 (Silent SL Placement Failure) from leaving positions unprotected * * Architecture: Event-driven verification with exponential backoff (30s, 60s, 90s) * - Queries Drift on-chain state, not just database * - 3 verification attempts with increasing delays * - If all fail: Halt trading + close position immediately * * Rate Limit Impact: ~3-9 queries per position (vs 360/hour with interval-based) * * Created: Dec 16, 2025 (Bug #76 root cause + user mandate "better safe than sorry") */ import { getDriftService } from '../drift/client' import { getMarketConfig } from '../../config/trading' import { closePosition } from '../drift/orders' import { updateTradeState } from '../database/trades' import { sendTelegramMessage } from '../notifications/telegram' import { OrderType, Order } from '@drift-labs/sdk' // Global trading halt flag let tradingHalted = false let haltReason = '' export function isTradingHalted(): boolean { return tradingHalted } export function getHaltReason(): string { return haltReason } export function resetTradingHalt(): void { tradingHalted = false haltReason = '' console.log('✅ Trading halt reset - system re-enabled') } /** * Query Drift on-chain state to verify SL orders exist * Returns true if at least one SL order found (TRIGGER_MARKET or TRIGGER_LIMIT) */ export async function querySLOrdersFromDrift( symbol: string, marketIndex: number ): Promise<{ exists: boolean; orderCount: number; orderTypes: string[] }> { try { const driftService = getDriftService() const driftClient = driftService.getClient() // Get open orders from the drift client const allOrders = driftClient.getUser().getOpenOrders() const marketOrders = allOrders.filter( (order: Order) => order.marketIndex === marketIndex && order.reduceOnly === true ) // Find SL orders (TRIGGER_MARKET or TRIGGER_LIMIT) const slOrders = marketOrders.filter( (order: Order) => order.orderType === OrderType.TRIGGER_MARKET || order.orderType === OrderType.TRIGGER_LIMIT ) const orderTypes = slOrders.map((o: Order) => { if (o.orderType === OrderType.TRIGGER_MARKET) return 'TRIGGER_MARKET' if (o.orderType === OrderType.TRIGGER_LIMIT) return 'TRIGGER_LIMIT' return 'UNKNOWN' }) console.log(`🔍 SL Verification for ${symbol}:`) console.log(` Total reduce-only orders: ${marketOrders.length}`) console.log(` SL orders found: ${slOrders.length}`) if (slOrders.length > 0) { console.log(` Order types: ${orderTypes.join(', ')}`) } return { exists: slOrders.length > 0, orderCount: slOrders.length, orderTypes, } } catch (error) { console.error(`❌ Error querying SL orders from Drift:`, error) // On error, assume SL might exist (fail-open for transient failures) return { exists: true, orderCount: 0, orderTypes: [] } } } /** * Attempt to place SL orders for a position that's missing them * Loads trade from database, calls placeExitOrders with SL-only params * Updates database with SL signatures if successful * * @returns true if SL orders placed successfully, false otherwise */ async function attemptSLPlacement( tradeId: string, symbol: string, marketIndex: number ): Promise { try { console.log(`🔧 Attempting to place SL orders for ${symbol} (Trade: ${tradeId})`) // Load trade from database const { getPrismaClient } = await import('../database/trades') const prisma = getPrismaClient() const trade = await prisma.trade.findUnique({ where: { id: tradeId }, select: { positionSizeUSD: true, entryPrice: true, stopLossPrice: true, direction: true, takeProfit1Price: true, takeProfit2Price: true, } }) if (!trade) { console.error(`❌ Trade ${tradeId} not found in database`) return false } if (!trade.stopLossPrice) { console.error(`❌ Trade ${tradeId} has no stopLossPrice set`) return false } // Import placeExitOrders dynamically const { placeExitOrders } = await import('../drift/orders') // Use stored TP prices, fallback to entry if missing const tp1Price = trade.takeProfit1Price || trade.entryPrice const tp2Price = trade.takeProfit2Price || trade.entryPrice // Place SL orders only (tp1SizePercent=0, tp2SizePercent=0) console.log(` Placing SL-only orders (preserving existing TP orders)`) const result = await placeExitOrders({ symbol, positionSizeUSD: trade.positionSizeUSD, entryPrice: trade.entryPrice, tp1Price: tp1Price, tp2Price: tp2Price, stopLossPrice: trade.stopLossPrice, tp1SizePercent: 0, // Don't place TP1 order tp2SizePercent: 0, // Don't place TP2 order direction: trade.direction as 'long' | 'short', }) if (!result.success) { console.error(`❌ placeExitOrders failed:`, result.error) return false } if (!result.signatures || result.signatures.length === 0) { console.error(`❌ placeExitOrders returned success but no signatures`) return false } // Update database with SL signatures const updateData: any = {} // Check if dual stops are being used (2 signatures) or single SL (1 signature) if (result.signatures.length === 2) { // Dual stops: soft SL + hard SL updateData.softStopOrderTx = result.signatures[0] updateData.hardStopOrderTx = result.signatures[1] console.log(` Placed 2 SL orders: soft (${result.signatures[0]}) + hard (${result.signatures[1]})`) } else { // Single SL updateData.slOrderTx = result.signatures[0] console.log(` Placed 1 SL order: ${result.signatures[0]}`) } await prisma.trade.update({ where: { id: tradeId }, data: updateData }) console.log(`✅ SL orders placed and database updated for ${symbol}`) return true } catch (error) { console.error(`❌ Error in attemptSLPlacement:`, error) return false } } /** * Halt trading and close position immediately * Called when SL verification fails after all retries */ async function haltTradingAndClosePosition( tradeId: string, symbol: string, reason: string ): Promise { try { // Set global halt flag tradingHalted = true haltReason = reason console.error(`🚨🚨🚨 TRADING HALTED 🚨🚨🚨`) console.error(` Reason: ${reason}`) console.error(` Trade ID: ${tradeId}`) console.error(` Symbol: ${symbol}`) console.error(` Action: Closing position immediately for safety`) // Send critical Telegram alert await sendTelegramMessage(`🚨🚨🚨 CRITICAL: TRADING HALTED 🚨🚨🚨 Reason: ${reason} Trade ID: ${tradeId} Symbol: ${symbol} Action Taken: ✅ Closing position immediately (safety) ⛔ New trades blocked until manual reset Position left unprotected - closing to prevent losses. Manual reset required: Check logs and reset via API or Telegram.`) // Close position immediately console.log(`🔒 Closing ${symbol} position for safety...`) const closeResult = await closePosition({ symbol, percentToClose: 100, slippageTolerance: 0.02, // 2% slippage tolerance for emergency closes }) if (closeResult.success) { console.log(`✅ Position closed successfully: ${closeResult.transactionSignature}`) console.log(` Emergency closure reason: ${reason}`) // Update database with emergency exit reason const { getPrismaClient } = await import('../database/trades') const prisma = getPrismaClient() await prisma.trade.update({ where: { id: tradeId }, data: { exitReason: 'emergency', } }) } else { console.error(`❌ Failed to close position: ${closeResult.error}`) await sendTelegramMessage(`❌ CRITICAL: Failed to close position ${symbol} Close Error: ${closeResult.error} MANUAL INTERVENTION REQUIRED IMMEDIATELY`) } } catch (error) { console.error(`❌ Error in haltTradingAndClosePosition:`, error) await sendTelegramMessage(`❌ CRITICAL: Error halting trading and closing position Error: ${error instanceof Error ? error.message : String(error)} MANUAL INTERVENTION REQUIRED IMMEDIATELY`) } } /** * Verify SL orders exist with hybrid approach: passive check → active recovery * - 30s: Passive check only (give initial placement time to propagate) * - 60s: Check + try to place SL if missing (active recovery attempt 1) * - 90s: Check + try to place SL if missing (active recovery attempt 2) * - If both 60s and 90s recovery attempts fail: Halt trading + close position * * Usage: Call after position opened successfully * Example: await verifySLWithRetries(tradeId, symbol, marketIndex) */ export async function verifySLWithRetries( tradeId: string, symbol: string, marketIndex: number ): Promise { const delays = [30000, 60000, 90000] // 30s, 60s, 90s console.log(`🛡️ Starting SL verification for ${symbol} (Trade: ${tradeId})`) console.log(` Strategy: 30s passive → 60s/90s active recovery if needed`) // ATTEMPT 1 (30s): Passive check only console.log(`⏱️ Verification attempt 1/3 (PASSIVE CHECK) - waiting 30s...`) await new Promise(resolve => setTimeout(resolve, delays[0])) let slStatus = await querySLOrdersFromDrift(symbol, marketIndex) if (slStatus.exists) { console.log(`✅ SL VERIFIED on attempt 1/3 (passive check)`) console.log(` Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`) return // Success - SL orders exist } console.error(`❌ SL NOT FOUND on attempt 1/3 - will attempt active recovery`) // ATTEMPT 2 (60s): Active recovery - try to place SL console.log(`⏱️ Verification attempt 2/3 (ACTIVE RECOVERY) - waiting 60s...`) await new Promise(resolve => setTimeout(resolve, delays[1])) // Check again first slStatus = await querySLOrdersFromDrift(symbol, marketIndex) if (slStatus.exists) { console.log(`✅ SL VERIFIED on attempt 2/3 (appeared naturally)`) console.log(` Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`) return // Success - SL orders now exist } console.error(`❌ SL still missing on attempt 2/3 - attempting to place SL orders...`) const recovery2Success = await attemptSLPlacement(tradeId, symbol, marketIndex) if (recovery2Success) { console.log(`✅ SL PLACED successfully on attempt 2/3 (active recovery)`) return // Success - SL orders placed } console.error(`❌ SL placement FAILED on attempt 2/3`) // ATTEMPT 3 (90s): Final active recovery - try to place SL again console.log(`⏱️ Verification attempt 3/3 (FINAL RECOVERY) - waiting 90s...`) await new Promise(resolve => setTimeout(resolve, delays[2])) // Check one more time slStatus = await querySLOrdersFromDrift(symbol, marketIndex) if (slStatus.exists) { console.log(`✅ SL VERIFIED on attempt 3/3 (appeared naturally)`) console.log(` Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`) return // Success - SL orders now exist } console.error(`❌ SL still missing on attempt 3/3 - final placement attempt...`) const recovery3Success = await attemptSLPlacement(tradeId, symbol, marketIndex) if (recovery3Success) { console.log(`✅ SL PLACED successfully on attempt 3/3 (final recovery)`) return // Success - SL orders placed } // BOTH RECOVERY ATTEMPTS FAILED - EMERGENCY SHUTDOWN console.error(`🚨 CRITICAL: SL verification + both recovery attempts FAILED`) console.error(` Symbol: ${symbol}`) console.error(` Trade ID: ${tradeId}`) console.error(` Action: Halting trading + closing position`) await haltTradingAndClosePosition( tradeId, symbol, `SL verification failed - both 60s and 90s recovery attempts unsuccessful` ) } /** * Check if trading is halted before accepting new trades * Returns { allow: boolean, reason: string } */ export function checkTradingAllowed(): { allow: boolean; reason: string } { if (tradingHalted) { return { allow: false, reason: `Trading halted: ${haltReason}. Manual reset required.`, } } return { allow: true, reason: '' } }