/** * Drift State Verifier Service * * Double-checks that positions marked as closed in our database * are actually closed on Drift Protocol. If mismatches found, * attempts to close the position again. * * Background: Drift occasionally confirms close transactions but * doesn't actually close the position (state propagation delay or * partial fill issues). This service detects and fixes those cases. * * Created: Dec 7, 2025 */ import { getDriftService } from '../drift/client' import { getPrismaClient } from '../database/trades' import { closePosition } from '../drift/orders' import { sendTelegramMessage } from '../notifications/telegram' import { Prisma } from '@prisma/client' export interface DriftStateMismatch { tradeId: string symbol: string expectedState: 'closed' | 'open' actualState: 'closed' | 'open' driftSize: number dbExitReason: string | null timeSinceExit: number // milliseconds } class DriftStateVerifier { private isRunning: boolean = false private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes private intervalId: NodeJS.Timeout | null = null // BUG #80 FIX: Track close attempts per symbol to enforce cooldown private recentCloseAttempts: Map = new Map() private readonly COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes /** * Start the periodic verification service */ start(): void { if (this.isRunning) { console.log('πŸ” Drift state verifier already running') return } console.log('πŸ” Starting Drift state verifier (checks every 10 minutes)') this.isRunning = true // Run first check after 2 minutes (allow time for initial startup) setTimeout(() => { this.runVerification().catch(err => { console.error('❌ Error in initial Drift state verification:', err) }) }, 2 * 60 * 1000) // Then run every 10 minutes this.intervalId = setInterval(() => { this.runVerification().catch(err => { console.error('❌ Error in Drift state verification:', err) }) }, this.checkIntervalMs) } /** * Stop the periodic verification service */ stop(): void { if (this.intervalId) { clearInterval(this.intervalId) this.intervalId = null } this.isRunning = false console.log('πŸ” Drift state verifier stopped') } /** * Run verification check once (can be called manually) */ async runVerification(): Promise { console.log('πŸ” Running Drift state verification...') const mismatches: DriftStateMismatch[] = [] try { const driftService = await getDriftService() const prisma = getPrismaClient() // Check 1: Find trades marked as closed in last 24 hours // These should definitely not exist on Drift anymore const recentlyClosedTrades = await prisma.trade.findMany({ where: { exitReason: { not: null }, exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } }, select: { id: true, positionId: true, symbol: true, exitReason: true, exitTime: true, }, }) console.log(` Checking ${recentlyClosedTrades.length} recently closed trades...`) for (const trade of recentlyClosedTrades) { try { // Extract market index from symbol (SOL-PERP β†’ 0, ETH-PERP β†’ 1, etc.) const marketIndex = this.getMarketIndex(trade.symbol) if (marketIndex === null) continue // Query Drift for position const driftPosition = await driftService.getPosition(marketIndex) if (driftPosition && Math.abs(driftPosition.size) >= 0.01) { // MISMATCH: DB says closed, Drift says open const timeSinceExit = Date.now() - new Date(trade.exitTime!).getTime() mismatches.push({ tradeId: trade.id, symbol: trade.symbol, expectedState: 'closed', actualState: 'open', driftSize: Math.abs(driftPosition.size), dbExitReason: trade.exitReason, timeSinceExit, }) console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) console.error(` DB: Closed ${(timeSinceExit / 60000).toFixed(1)}min ago (${trade.exitReason})`) console.error(` Drift: Still open with size ${driftPosition.size}`) } } catch (err) { console.error(` Error checking ${trade.symbol}:`, err) } } // Check 2: Find trades marked as open but actually closed on Drift // (Less critical but worth detecting) const openTrades = await prisma.trade.findMany({ where: { exitReason: null, }, select: { id: true, positionId: true, symbol: true, createdAt: true, }, }) console.log(` Checking ${openTrades.length} open trades...`) for (const trade of openTrades) { try { const marketIndex = this.getMarketIndex(trade.symbol) if (marketIndex === null) continue const driftPosition = await driftService.getPosition(marketIndex) if (!driftPosition || Math.abs(driftPosition.size) < 0.01) { // MISMATCH: DB says open, Drift says closed const timeSinceExit = Date.now() - new Date(trade.createdAt).getTime() mismatches.push({ tradeId: trade.id, symbol: trade.symbol, expectedState: 'open', actualState: 'closed', driftSize: 0, dbExitReason: null, timeSinceExit, }) console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) console.error(` DB: Open since ${(timeSinceExit / 60000).toFixed(1)}min ago`) console.error(` Drift: Position closed (size 0)`) } } catch (err) { console.error(` Error checking ${trade.symbol}:`, err) } } if (mismatches.length === 0) { console.log(' βœ… No mismatches found - DB and Drift states match') } else { console.error(` ❌ Found ${mismatches.length} mismatches!`) await this.handleMismatches(mismatches) } } catch (error) { console.error('❌ Error running Drift state verification:', error) } return mismatches } /** * Handle detected mismatches */ private async handleMismatches(mismatches: DriftStateMismatch[]): Promise { for (const mismatch of mismatches) { if (mismatch.expectedState === 'closed' && mismatch.actualState === 'open') { // CRITICAL: Position should be closed but is still open on Drift await this.retryClose(mismatch) } else if (mismatch.expectedState === 'open' && mismatch.actualState === 'closed') { // Position closed externally - this is handled by Position Manager's ghost detection console.log(` ℹ️ ${mismatch.symbol}: Ghost position (will be cleaned by Position Manager)`) } } // Send Telegram alert await this.sendMismatchAlert(mismatches) } /** * Retry closing a position that should be closed but isn't * BUG #82 LONG-TERM FIX (Dec 10, 2025): Comprehensive position verification * * CRITICAL SAFETY CHECKS: * 1. Verify Drift position exists and matches DB record * 2. Check position freshness: is it NEWER than DB exit time? * 3. Verify size/direction alignment within tolerance * 4. Grace period: wait 10+ minutes after DB exit before acting * 5. Fail-open bias: when in doubt, do nothing and alert */ private async retryClose(mismatch: DriftStateMismatch): Promise { console.log(`πŸ”„ Analyzing close candidate for ${mismatch.symbol}...`) try { // STEP 1: Cooldown enforcement (prevents retry spam) const cooldownCheck = await this.checkCooldown(mismatch.symbol) if (!cooldownCheck.canProceed) { console.log(` ⏸️ ${cooldownCheck.reason}`) return } // STEP 2: Load full trade context from database const prisma = getPrismaClient() const dbTrade = await prisma.trade.findUnique({ where: { id: mismatch.tradeId }, select: { id: true, symbol: true, direction: true, entryTime: true, exitTime: true, exitReason: true, positionSizeUSD: true, entryPrice: true, configSnapshot: true, } }) if (!dbTrade) { console.warn(` ⚠️ SAFETY: Trade ${mismatch.tradeId} not found in DB - skipping`) return } // STEP 3: Verify Drift position exists and get full details const driftService = await getDriftService() const marketIndex = this.getMarketIndex(dbTrade.symbol) if (marketIndex === null) { console.warn(` ⚠️ SAFETY: Unknown market ${dbTrade.symbol} - skipping`) return } const driftPosition = await driftService.getPosition(marketIndex) if (!driftPosition || Math.abs(driftPosition.size) < 0.01) { console.log(` βœ… RESOLVED: Position already closed on Drift`) return } // STEP 4: CRITICAL VERIFICATION - Check if this is a NEW position const verificationResult = await this.verifyPositionIdentity({ dbTrade, driftPosition, mismatch, }) console.log(`\n πŸ“Š VERIFICATION DECISION:`, JSON.stringify(verificationResult, null, 2)) if (!verificationResult.isOldGhost) { console.warn(` ⚠️ PROTECTION TRIGGERED: ${verificationResult.reason}`) console.warn(` πŸ›‘οΈ Skipping close to protect potentially active position`) // Log detailed protection event await this.logProtectedPosition({ tradeId: dbTrade.id, symbol: dbTrade.symbol, reason: verificationResult.reason, details: verificationResult.details, }) return } // STEP 5: All checks passed - proceed with close console.log(` βœ… VERIFIED OLD GHOST: Safe to close`) console.log(` πŸ“‹ Evidence:`, verificationResult.details) const attemptTime = Date.now() this.recentCloseAttempts.set(dbTrade.symbol, attemptTime) const result = await closePosition({ symbol: dbTrade.symbol, percentToClose: 100, slippageTolerance: 0.05, }) if (result.success) { console.log(` βœ… Orphan closed: ${result.transactionSignature}`) console.log(` πŸ’° P&L: $${result.realizedPnL?.toFixed(2) || 0}`) // Record successful cleanup await prisma.trade.update({ where: { id: dbTrade.id }, data: { exitOrderTx: result.transactionSignature || 'ORPHAN_CLEANUP', realizedPnL: result.realizedPnL || 0, configSnapshot: { ...dbTrade.configSnapshot as any, orphanCleanup: true, orphanCleanupTime: new Date(attemptTime).toISOString(), verificationPassed: verificationResult.details, } } }) } else { console.error(` ❌ Close failed: ${result.error}`) } } catch (error) { console.error(` ❌ Error in close verification:`, error) this.recentCloseAttempts.set(mismatch.symbol, Date.now()) } } /** * Check cooldown status for a symbol */ private async checkCooldown(symbol: string): Promise<{ canProceed: boolean; reason?: string }> { // Check in-memory cooldown first const lastAttemptTime = this.recentCloseAttempts.get(symbol) if (lastAttemptTime) { const timeSinceAttempt = Date.now() - lastAttemptTime if (timeSinceAttempt < this.COOLDOWN_MS) { const remaining = Math.ceil((this.COOLDOWN_MS - timeSinceAttempt) / 1000) return { canProceed: false, reason: `Cooldown active: ${remaining}s remaining (last attempt ${(timeSinceAttempt/1000).toFixed(0)}s ago)` } } } // Check database cooldown (survives restarts) const prisma = getPrismaClient() const recentAttempt = await prisma.trade.findFirst({ where: { symbol, configSnapshot: { path: ['orphanCleanupTime'], not: Prisma.JsonNull, } }, orderBy: { updatedAt: 'desc' }, select: { configSnapshot: true } }) if (recentAttempt?.configSnapshot) { const snapshot = recentAttempt.configSnapshot as any const lastCleanup = snapshot.orphanCleanupTime ? new Date(snapshot.orphanCleanupTime) : null if (lastCleanup) { const timeSince = Date.now() - lastCleanup.getTime() if (timeSince < this.COOLDOWN_MS) { return { canProceed: false, reason: `Database cooldown: ${Math.ceil((this.COOLDOWN_MS - timeSince)/1000)}s remaining` } } } } return { canProceed: true } } /** * CRITICAL: Verify if Drift position is an old ghost or new active trade * * Uses multiple verification methods: * 1. Time-based: Position age vs DB exit time * 2. Size-based: Position size vs DB recorded size * 3. Grace period: Wait 10+ minutes after DB exit * 4. Direction check: Must match DB direction * * FAIL-OPEN BIAS: When verification is uncertain, assume position is active */ private async verifyPositionIdentity(params: { dbTrade: any driftPosition: any mismatch: DriftStateMismatch }): Promise<{ isOldGhost: boolean reason: string details: Record }> { const { dbTrade, driftPosition, mismatch } = params // Grace period check: Has enough time passed since DB exit? const GRACE_PERIOD_MS = 10 * 60 * 1000 // 10 minutes const timeSinceExit = Date.now() - new Date(dbTrade.exitTime).getTime() if (timeSinceExit < GRACE_PERIOD_MS) { return { isOldGhost: false, reason: 'GRACE_PERIOD_ACTIVE', details: { timeSinceExitMin: (timeSinceExit / 60000).toFixed(1), gracePeriodMin: 10, message: 'Too soon after exit - may still be propagating' } } } // Direction check: Must match const driftDirection = driftPosition.side // 'long' | 'short' | 'none' if (driftDirection !== dbTrade.direction) { return { isOldGhost: false, reason: 'DIRECTION_MISMATCH', details: { dbDirection: dbTrade.direction, driftDirection, message: 'Different direction = definitely different position' } } } // Size check: Must be within 15% tolerance // (Allows for partial fills, funding rate impacts, etc.) const dbSizeTokens = dbTrade.positionSizeUSD / dbTrade.entryPrice const driftSizeTokens = driftPosition.size const sizeRatio = Math.abs(driftSizeTokens) / Math.abs(dbSizeTokens) if (sizeRatio < 0.85 || sizeRatio > 1.15) { return { isOldGhost: false, reason: 'SIZE_MISMATCH', details: { dbSizeTokens: dbSizeTokens.toFixed(2), driftSizeTokens: driftSizeTokens.toFixed(2), sizeRatio: sizeRatio.toFixed(3), tolerance: '0.85-1.15', message: 'Size difference too large = likely different position' } } } // Position age estimation (best effort - no direct timestamp from SDK) // We use entry price comparison as a proxy: // - If Drift entry price significantly different from DB β†’ likely new position const priceDiffPercent = Math.abs(driftPosition.entryPrice - dbTrade.entryPrice) / dbTrade.entryPrice * 100 if (priceDiffPercent > 2.0) { return { isOldGhost: false, reason: 'ENTRY_PRICE_MISMATCH', details: { dbEntryPrice: dbTrade.entryPrice.toFixed(2), driftEntryPrice: driftPosition.entryPrice.toFixed(2), diffPercent: priceDiffPercent.toFixed(2), message: 'Entry price difference >2% suggests different position' } } } // Check if there are any newer trades on this symbol in DB const prisma = getPrismaClient() const newerTrades = await prisma.trade.findMany({ where: { symbol: dbTrade.symbol, exitReason: null, // Open trades createdAt: { gt: new Date(dbTrade.exitTime) } }, select: { id: true, createdAt: true } }) if (newerTrades.length > 0) { return { isOldGhost: false, reason: 'NEWER_TRADE_EXISTS', details: { newerTradeCount: newerTrades.length, newerTradeIds: newerTrades.map(t => t.id), message: 'DB shows newer open position on this symbol - Drift position likely belongs to it' } } } // ALL CHECKS PASSED - This appears to be an old ghost return { isOldGhost: true, reason: 'VERIFIED_OLD_GHOST', details: { timeSinceExitMin: (timeSinceExit / 60000).toFixed(1), directionMatch: true, sizeRatio: sizeRatio.toFixed(3), entryPriceDiff: priceDiffPercent.toFixed(2) + '%', noNewerTrades: true, message: 'All verification checks passed - safe to close' } } } /** * Log when we protect a position from accidental closure */ private async logProtectedPosition(params: { tradeId: string symbol: string reason: string details: Record }): Promise { try { const prisma = getPrismaClient() await prisma.trade.update({ where: { id: params.tradeId }, data: { configSnapshot: { path: ['protectionEvents'], arrayAppend: { timestamp: new Date().toISOString(), reason: params.reason, details: params.details, message: 'Position protected from accidental closure by Bug #82 fix' } } } }).catch(() => { // Ignore errors updating protection log - not critical }) } catch (error) { // Silent failure - protection logging is supplementary } } /** * Send Telegram alert about mismatches */ private async sendMismatchAlert(mismatches: DriftStateMismatch[]): Promise { const criticalMismatches = mismatches.filter(m => m.expectedState === 'closed' && m.actualState === 'open' ) if (criticalMismatches.length === 0) return const message = ` 🚨 DRIFT STATE MISMATCH ALERT Found ${criticalMismatches.length} position(s) that should be closed but are still open on Drift: ${criticalMismatches.map(m => ` πŸ“Š ${m.symbol} DB Status: Closed (${m.dbExitReason}) Drift Status: Open (${m.driftSize.toFixed(2)} tokens) Time since exit: ${(m.timeSinceExit / 60000).toFixed(1)} minutes ⚠️ Retry close attempted automatically `).join('\n')} This indicates Drift Protocol state propagation issues. Check Drift UI to verify actual position status. `.trim() try { await sendTelegramMessage(message) } catch (error) { console.error('Failed to send Telegram alert:', error) } } /** * Get Drift market index from symbol */ private getMarketIndex(symbol: string): number | null { const marketMap: Record = { 'SOL-PERP': 0, 'BTC-PERP': 1, 'ETH-PERP': 2, } return marketMap[symbol] ?? null } } // Singleton instance let verifierInstance: DriftStateVerifier | null = null export function getDriftStateVerifier(): DriftStateVerifier { if (!verifierInstance) { verifierInstance = new DriftStateVerifier() } return verifierInstance } export function startDriftStateVerifier(): void { const verifier = getDriftStateVerifier() verifier.start() }