trading_bot_v4/lib/safety/sl-verification.ts

/**
 * SL Verification System - Post-Open Safety Checks
 * 
 * Purpose: Verify stop loss orders are actually placed on-chain after position opens
 * Prevents Bug #76 (Silent SL Placement Failure) from leaving positions unprotected
 * 
 * Architecture: Event-driven verification with exponential backoff (30s, 60s, 90s)
 * - Queries Drift on-chain state, not just database
 * - 3 verification attempts with increasing delays
 * - If all fail: Halt trading + close position immediately
 * 
 * Rate Limit Impact: ~3-9 queries per position (vs 360/hour with interval-based)
 * 
 * Created: Dec 16, 2025 (Bug #76 root cause + user mandate "better safe than sorry")
 */

import { getDriftService } from '../drift/client'
import { getMarketConfig } from '../../config/trading'
import { closePosition } from '../drift/orders'
import { updateTradeState } from '../database/trades'
import { sendTelegramMessage } from '../notifications/telegram'
import { OrderType, Order } from '@drift-labs/sdk'

// Global trading halt flag
let tradingHalted = false
let haltReason = ''

export function isTradingHalted(): boolean {
  return tradingHalted
}

export function getHaltReason(): string {
  return haltReason
}

export function resetTradingHalt(): void {
  tradingHalted = false
  haltReason = ''
  console.log('✅ Trading halt reset - system re-enabled')
}

/**
 * Query Drift on-chain state to verify SL orders exist
 * Returns true if at least one SL order found (TRIGGER_MARKET or TRIGGER_LIMIT)
 */
export async function querySLOrdersFromDrift(
  symbol: string,
  marketIndex: number
): Promise<{ exists: boolean; orderCount: number; orderTypes: string[] }> {
  try {
    const driftService = getDriftService()
    const driftClient = driftService.getClient()
    
    // Get open orders from the drift client
    const allOrders = driftClient.getUser().getOpenOrders()
    
    const marketOrders = allOrders.filter(
      (order: Order) => order.marketIndex === marketIndex && order.reduceOnly === true
    )
    
    // Find SL orders (TRIGGER_MARKET or TRIGGER_LIMIT)
    const slOrders = marketOrders.filter(
      (order: Order) => 
        order.orderType === OrderType.TRIGGER_MARKET ||
        order.orderType === OrderType.TRIGGER_LIMIT
    )
    
    const orderTypes = slOrders.map((o: Order) => {
      if (o.orderType === OrderType.TRIGGER_MARKET) return 'TRIGGER_MARKET'
      if (o.orderType === OrderType.TRIGGER_LIMIT) return 'TRIGGER_LIMIT'
      return 'UNKNOWN'
    })
    
    console.log(`🔍 SL Verification for ${symbol}:`)
    console.log(`   Total reduce-only orders: ${marketOrders.length}`)
    console.log(`   SL orders found: ${slOrders.length}`)
    if (slOrders.length > 0) {
      console.log(`   Order types: ${orderTypes.join(', ')}`)
    }
    
    return {
      exists: slOrders.length > 0,
      orderCount: slOrders.length,
      orderTypes,
    }
  } catch (error) {
    console.error(`❌ Error querying SL orders from Drift:`, error)
    // On error, assume SL might exist (fail-open for transient failures)
    return { exists: true, orderCount: 0, orderTypes: [] }
  }
}

/**
 * Attempt to place SL orders for a position that's missing them
 * Loads trade from database, calls placeExitOrders with SL-only params
 * Updates database with SL signatures if successful
 * 
 * @returns true if SL orders placed successfully, false otherwise
 */
async function attemptSLPlacement(
  tradeId: string,
  symbol: string,
  marketIndex: number
): Promise<boolean> {
  try {
    console.log(`🔧 Attempting to place SL orders for ${symbol} (Trade: ${tradeId})`)
    
    // Load trade from database
    const { getPrismaClient } = await import('../database/trades')
    const prisma = getPrismaClient()
    const trade = await prisma.trade.findUnique({
      where: { id: tradeId },
      select: {
        positionSizeUSD: true,
        entryPrice: true,
        stopLossPrice: true,
        direction: true,
        takeProfit1Price: true,
        takeProfit2Price: true,
      }
    })
    
    if (!trade) {
      console.error(`❌ Trade ${tradeId} not found in database`)
      return false
    }
    
    if (!trade.stopLossPrice) {
      console.error(`❌ Trade ${tradeId} has no stopLossPrice set`)
      return false
    }
    
    // Import placeExitOrders dynamically
    const { placeExitOrders } = await import('../drift/orders')
    
    // Use stored TP prices, fallback to entry if missing
    const tp1Price = trade.takeProfit1Price || trade.entryPrice
    const tp2Price = trade.takeProfit2Price || trade.entryPrice
    
    // Place SL orders only (tp1SizePercent=0, tp2SizePercent=0)
    console.log(`   Placing SL-only orders (preserving existing TP orders)`)
    const result = await placeExitOrders({
      symbol,
      positionSizeUSD: trade.positionSizeUSD,
      entryPrice: trade.entryPrice,
      tp1Price: tp1Price,
      tp2Price: tp2Price,
      stopLossPrice: trade.stopLossPrice,
      tp1SizePercent: 0, // Don't place TP1 order
      tp2SizePercent: 0, // Don't place TP2 order
      direction: trade.direction as 'long' | 'short',
    })
    
    if (!result.success) {
      console.error(`❌ placeExitOrders failed:`, result.error)
      return false
    }
    
    if (!result.signatures || result.signatures.length === 0) {
      console.error(`❌ placeExitOrders returned success but no signatures`)
      return false
    }
    
    // Update database with SL signatures
    const updateData: any = {}
    
    // Check if dual stops are being used (2 signatures) or single SL (1 signature)
    if (result.signatures.length === 2) {
      // Dual stops: soft SL + hard SL
      updateData.softStopOrderTx = result.signatures[0]
      updateData.hardStopOrderTx = result.signatures[1]
      console.log(`   Placed 2 SL orders: soft (${result.signatures[0]}) + hard (${result.signatures[1]})`)
    } else {
      // Single SL
      updateData.slOrderTx = result.signatures[0]
      console.log(`   Placed 1 SL order: ${result.signatures[0]}`)
    }
    
    await prisma.trade.update({
      where: { id: tradeId },
      data: updateData
    })
    
    console.log(`✅ SL orders placed and database updated for ${symbol}`)
    return true
    
  } catch (error) {
    console.error(`❌ Error in attemptSLPlacement:`, error)
    return false
  }
}

/**
 * Halt trading and close position immediately
 * Called when SL verification fails after all retries
 */
async function haltTradingAndClosePosition(
  tradeId: string,
  symbol: string,
  reason: string
): Promise<void> {
  try {
    // Set global halt flag
    tradingHalted = true
    haltReason = reason
    
    console.error(`🚨🚨🚨 TRADING HALTED 🚨🚨🚨`)
    console.error(`   Reason: ${reason}`)
    console.error(`   Trade ID: ${tradeId}`)
    console.error(`   Symbol: ${symbol}`)
    console.error(`   Action: Closing position immediately for safety`)
    
    // Send critical Telegram alert
    await sendTelegramMessage(`🚨🚨🚨 CRITICAL: TRADING HALTED 🚨🚨🚨

Reason: ${reason}

Trade ID: ${tradeId}
Symbol: ${symbol}

Action Taken:
✅ Closing position immediately (safety)
⛔ New trades blocked until manual reset

Position left unprotected - closing to prevent losses.

Manual reset required: Check logs and reset via API or Telegram.`)
    
    // Close position immediately
    console.log(`🔒 Closing ${symbol} position for safety...`)
    const closeResult = await closePosition({
      symbol,
      percentToClose: 100,
      slippageTolerance: 0.02, // 2% slippage tolerance for emergency closes
    })
    
    if (closeResult.success) {
      console.log(`✅ Position closed successfully: ${closeResult.transactionSignature}`)
      console.log(`   Emergency closure reason: ${reason}`)
      
      // Update database with emergency exit reason
      const { getPrismaClient } = await import('../database/trades')
      const prisma = getPrismaClient()
      await prisma.trade.update({
        where: { id: tradeId },
        data: {
          exitReason: 'emergency',
        }
      })
    } else {
      console.error(`❌ Failed to close position: ${closeResult.error}`)
      await sendTelegramMessage(`❌ CRITICAL: Failed to close position ${symbol}

Close Error: ${closeResult.error}

MANUAL INTERVENTION REQUIRED IMMEDIATELY`)
    }
  } catch (error) {
    console.error(`❌ Error in haltTradingAndClosePosition:`, error)
    await sendTelegramMessage(`❌ CRITICAL: Error halting trading and closing position

Error: ${error instanceof Error ? error.message : String(error)}

MANUAL INTERVENTION REQUIRED IMMEDIATELY`)
  }
}

/**
 * Verify SL orders exist with hybrid approach: passive check → active recovery
 * - 30s: Passive check only (give initial placement time to propagate)
 * - 60s: Check + try to place SL if missing (active recovery attempt 1)
 * - 90s: Check + try to place SL if missing (active recovery attempt 2)
 * - If both 60s and 90s recovery attempts fail: Halt trading + close position
 * 
 * Usage: Call after position opened successfully
 * Example: await verifySLWithRetries(tradeId, symbol, marketIndex)
 */
export async function verifySLWithRetries(
  tradeId: string,
  symbol: string,
  marketIndex: number
): Promise<void> {
  const delays = [30000, 60000, 90000] // 30s, 60s, 90s
  
  console.log(`🛡️ Starting SL verification for ${symbol} (Trade: ${tradeId})`)
  console.log(`   Strategy: 30s passive → 60s/90s active recovery if needed`)
  
  // ATTEMPT 1 (30s): Passive check only
  console.log(`⏱️ Verification attempt 1/3 (PASSIVE CHECK) - waiting 30s...`)
  await new Promise(resolve => setTimeout(resolve, delays[0]))
  
  let slStatus = await querySLOrdersFromDrift(symbol, marketIndex)
  
  if (slStatus.exists) {
    console.log(`✅ SL VERIFIED on attempt 1/3 (passive check)`)
    console.log(`   Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`)
    return // Success - SL orders exist
  }
  
  console.error(`❌ SL NOT FOUND on attempt 1/3 - will attempt active recovery`)
  
  // ATTEMPT 2 (60s): Active recovery - try to place SL
  console.log(`⏱️ Verification attempt 2/3 (ACTIVE RECOVERY) - waiting 60s...`)
  await new Promise(resolve => setTimeout(resolve, delays[1]))
  
  // Check again first
  slStatus = await querySLOrdersFromDrift(symbol, marketIndex)
  
  if (slStatus.exists) {
    console.log(`✅ SL VERIFIED on attempt 2/3 (appeared naturally)`)
    console.log(`   Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`)
    return // Success - SL orders now exist
  }
  
  console.error(`❌ SL still missing on attempt 2/3 - attempting to place SL orders...`)
  const recovery2Success = await attemptSLPlacement(tradeId, symbol, marketIndex)
  
  if (recovery2Success) {
    console.log(`✅ SL PLACED successfully on attempt 2/3 (active recovery)`)
    return // Success - SL orders placed
  }
  
  console.error(`❌ SL placement FAILED on attempt 2/3`)
  
  // ATTEMPT 3 (90s): Final active recovery - try to place SL again
  console.log(`⏱️ Verification attempt 3/3 (FINAL RECOVERY) - waiting 90s...`)
  await new Promise(resolve => setTimeout(resolve, delays[2]))
  
  // Check one more time
  slStatus = await querySLOrdersFromDrift(symbol, marketIndex)
  
  if (slStatus.exists) {
    console.log(`✅ SL VERIFIED on attempt 3/3 (appeared naturally)`)
    console.log(`   Found ${slStatus.orderCount} SL order(s): ${slStatus.orderTypes.join(', ')}`)
    return // Success - SL orders now exist
  }
  
  console.error(`❌ SL still missing on attempt 3/3 - final placement attempt...`)
  const recovery3Success = await attemptSLPlacement(tradeId, symbol, marketIndex)
  
  if (recovery3Success) {
    console.log(`✅ SL PLACED successfully on attempt 3/3 (final recovery)`)
    return // Success - SL orders placed
  }
  
  // BOTH RECOVERY ATTEMPTS FAILED - EMERGENCY SHUTDOWN
  console.error(`🚨 CRITICAL: SL verification + both recovery attempts FAILED`)
  console.error(`   Symbol: ${symbol}`)
  console.error(`   Trade ID: ${tradeId}`)
  console.error(`   Action: Halting trading + closing position`)
  
  await haltTradingAndClosePosition(
    tradeId,
    symbol,
    `SL verification failed - both 60s and 90s recovery attempts unsuccessful`
  )
}

/**
 * Check if trading is halted before accepting new trades
 * Returns { allow: boolean, reason: string }
 */
export function checkTradingAllowed(): { allow: boolean; reason: string } {
  if (tradingHalted) {
    return {
      allow: false,
      reason: `Trading halted: ${haltReason}. Manual reset required.`,
    }
  }
  
  return { allow: true, reason: '' }
}