/** * Database-Drift Synchronization Validator * * Periodically validates that database "open" trades match actual Drift positions * Runs independently of Position Manager to catch ghost positions * * Ghost positions occur when: * - On-chain orders fill but database update fails * - Position Manager closes position but DB write fails * - Container restarts before cleanup completes * * Created: November 16, 2025 */ import { getPrismaClient } from './trades' import { initializeDriftService } from '../drift/client' import { getMarketConfig } from '../../config/trading' let validationInterval: NodeJS.Timeout | null = null let isRunning = false interface ValidationResult { checked: number ghosts: number orphans: number valid: number errors: string[] } /** * Start periodic validation (runs every 10 minutes) */ export function startDatabaseSyncValidator(): void { if (validationInterval) { console.log('⚠️ Database sync validator already running') return } // Run immediately on start setTimeout(() => validateAllOpenTrades(), 5000) // 5s delay to let system initialize // Then run every 10 minutes validationInterval = setInterval(async () => { await validateAllOpenTrades() }, 10 * 60 * 1000) console.log('🔍 Database sync validator started (runs every 10 minutes)') } /** * Stop periodic validation */ export function stopDatabaseSyncValidator(): void { if (validationInterval) { clearInterval(validationInterval) validationInterval = null console.log('🛑 Database sync validator stopped') } } /** * Validate all "open" trades in database against Drift positions * * This is the master validation that ensures database accuracy */ export async function validateAllOpenTrades(): Promise { if (isRunning) { console.log('⏭️ Validation already in progress, skipping...') return { checked: 0, ghosts: 0, orphans: 0, valid: 0, errors: [] } } isRunning = true const result: ValidationResult = { checked: 0, ghosts: 0, orphans: 0, valid: 0, errors: [] } try { const prisma = getPrismaClient() // Get all trades marked as "open" in database const openTrades = await prisma.trade.findMany({ where: { exitReason: null }, orderBy: { createdAt: 'desc' } }) if (openTrades.length === 0) { console.log('✅ No open trades to validate') isRunning = false return result } console.log(`🔍 Validating ${openTrades.length} open trades against Drift...`) result.checked = openTrades.length // Initialize Drift service let driftService try { driftService = await initializeDriftService() } catch (error) { const errorMsg = `Failed to initialize Drift service: ${error}` console.error(`❌ ${errorMsg}`) result.errors.push(errorMsg) isRunning = false return result } // Get all Drift positions (one API call) let driftPositions try { driftPositions = await driftService.getAllPositions() console.log(`📊 Found ${driftPositions.length} actual positions on Drift`) } catch (error) { const errorMsg = `Failed to fetch Drift positions: ${error}` console.error(`❌ ${errorMsg}`) result.errors.push(errorMsg) isRunning = false return result } // Group database trades by symbol to handle multiple DB entries for same position const tradesBySymbol = new Map() for (const trade of openTrades) { const existing = tradesBySymbol.get(trade.symbol) || [] existing.push(trade) tradesBySymbol.set(trade.symbol, existing) } // Validate each symbol's trades against Drift position for (const [symbol, trades] of tradesBySymbol) { try { await validateSymbolTrades(symbol, trades, driftPositions, result) } catch (error) { const errorMsg = `Error validating ${symbol}: ${error}` console.error(`❌ ${errorMsg}`) result.errors.push(errorMsg) } } // Log summary console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') console.log('📊 DATABASE SYNC VALIDATION COMPLETE') console.log(` Checked: ${result.checked} trades`) console.log(` ✅ Valid: ${result.valid} (DB matches Drift)`) console.log(` 👻 Ghosts: ${result.ghosts} (DB open, Drift closed) - FIXED`) console.log(` 🔄 Orphans: ${result.orphans} (DB closed, Drift open) - FIXED`) if (result.errors.length > 0) { console.log(` ⚠️ Errors: ${result.errors.length}`) } console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') } catch (error) { console.error('❌ Database sync validation failed:', error) result.errors.push(`Validation failed: ${error}`) } finally { isRunning = false } return result } /** * Validate all trades for a symbol against the Drift position * * CRITICAL: Multiple DB trades can exist for one Drift position * This happens because old trades get re-opened by startup validator * Only the MOST RECENT trade should be kept open */ async function validateSymbolTrades( symbol: string, trades: any[], driftPositions: any[], result: ValidationResult ): Promise { const prisma = getPrismaClient() // Find matching Drift position const driftPosition = driftPositions.find(p => p.symbol === symbol) if (!driftPosition || Math.abs(driftPosition.size) < 0.01) { // NO POSITION ON DRIFT - all DB trades for this symbol are ghosts console.log(`👻 GHOST DETECTED: ${symbol} - ${trades.length} DB trades but no Drift position`) for (const trade of trades) { const hasRealizedPnL = trade.realizedPnL !== null && trade.realizedPnL !== undefined const exitReason = hasRealizedPnL ? 'manual' : 'GHOST_CLEANUP' const exitPrice = trade.exitPrice || trade.entryPrice const realizedPnL = trade.realizedPnL || 0 await prisma.trade.update({ where: { id: trade.id }, data: { exitReason: exitReason, exitTime: new Date(), exitPrice: exitPrice, realizedPnL: realizedPnL, status: 'closed' } }) console.log(` ✅ Closed ghost trade ${trade.id} (${exitReason})`) result.ghosts++ } return } // POSITION EXISTS ON DRIFT const driftDirection = driftPosition.side.toLowerCase() // Check if any trade has wrong direction const wrongDirection = trades.find(t => t.direction !== driftDirection) if (wrongDirection) { console.log(`⚠️ DIRECTION MISMATCH: ${symbol}`) console.log(` DB: ${wrongDirection.direction} | Drift: ${driftDirection}`) result.errors.push(`${symbol}: Direction mismatch`) return } // If multiple trades exist for same symbol, keep only the MOST RECENT if (trades.length > 1) { console.log(`🔄 DUPLICATE TRADES: ${symbol} has ${trades.length} open DB entries for 1 Drift position`) // Sort by creation time, keep newest trades.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()) const keepTrade = trades[0] const closeTrades = trades.slice(1) console.log(` ✅ Keeping: ${keepTrade.id} (${keepTrade.createdAt.toISOString()})`) for (const trade of closeTrades) { const hasRealizedPnL = trade.realizedPnL !== null && trade.realizedPnL !== undefined const exitReason = hasRealizedPnL ? 'manual' : 'DUPLICATE_CLEANUP' await prisma.trade.update({ where: { id: trade.id }, data: { exitReason: exitReason, exitTime: new Date(), exitPrice: trade.exitPrice || trade.entryPrice, realizedPnL: trade.realizedPnL || 0, status: 'closed' } }) console.log(` 🗑️ Closed duplicate: ${trade.id} (${trade.createdAt.toISOString()})`) result.ghosts++ } result.valid++ // Count the one we kept } else { // Single trade matches single Drift position - all good result.valid++ } } /** * One-time manual validation (for API endpoint or debugging) */ export async function runManualValidation(): Promise { console.log('🔧 Running manual database validation...') return await validateAllOpenTrades() }