/** * Drift State Verifier Service * * Double-checks that positions marked as closed in our database * are actually closed on Drift Protocol. If mismatches found, * attempts to close the position again. * * Background: Drift occasionally confirms close transactions but * doesn't actually close the position (state propagation delay or * partial fill issues). This service detects and fixes those cases. * * Created: Dec 7, 2025 */ import { getDriftService } from '../drift/client' import { getPrismaClient } from '../database/trades' import { closePosition } from '../drift/orders' import { sendTelegramMessage } from '../notifications/telegram' export interface DriftStateMismatch { tradeId: string symbol: string expectedState: 'closed' | 'open' actualState: 'closed' | 'open' driftSize: number dbExitReason: string | null timeSinceExit: number // milliseconds } class DriftStateVerifier { private isRunning: boolean = false private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes private intervalId: NodeJS.Timeout | null = null /** * Start the periodic verification service */ start(): void { if (this.isRunning) { console.log('🔍 Drift state verifier already running') return } console.log('🔍 Starting Drift state verifier (checks every 10 minutes)') this.isRunning = true // Run first check after 2 minutes (allow time for initial startup) setTimeout(() => { this.runVerification().catch(err => { console.error('❌ Error in initial Drift state verification:', err) }) }, 2 * 60 * 1000) // Then run every 10 minutes this.intervalId = setInterval(() => { this.runVerification().catch(err => { console.error('❌ Error in Drift state verification:', err) }) }, this.checkIntervalMs) } /** * Stop the periodic verification service */ stop(): void { if (this.intervalId) { clearInterval(this.intervalId) this.intervalId = null } this.isRunning = false console.log('🔍 Drift state verifier stopped') } /** * Run verification check once (can be called manually) */ async runVerification(): Promise { console.log('🔍 Running Drift state verification...') const mismatches: DriftStateMismatch[] = [] try { const driftService = await getDriftService() const prisma = getPrismaClient() // Check 1: Find trades marked as closed in last 24 hours // These should definitely not exist on Drift anymore const recentlyClosedTrades = await prisma.trade.findMany({ where: { exitReason: { not: null }, exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } }, select: { id: true, positionId: true, symbol: true, exitReason: true, exitTime: true, }, }) console.log(` Checking ${recentlyClosedTrades.length} recently closed trades...`) for (const trade of recentlyClosedTrades) { try { // Extract market index from symbol (SOL-PERP → 0, ETH-PERP → 1, etc.) const marketIndex = this.getMarketIndex(trade.symbol) if (marketIndex === null) continue // Query Drift for position const driftPosition = await driftService.getPosition(marketIndex) if (driftPosition && Math.abs(driftPosition.size) >= 0.01) { // MISMATCH: DB says closed, Drift says open const timeSinceExit = Date.now() - new Date(trade.exitTime!).getTime() mismatches.push({ tradeId: trade.id, symbol: trade.symbol, expectedState: 'closed', actualState: 'open', driftSize: Math.abs(driftPosition.size), dbExitReason: trade.exitReason, timeSinceExit, }) console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) console.error(` DB: Closed ${(timeSinceExit / 60000).toFixed(1)}min ago (${trade.exitReason})`) console.error(` Drift: Still open with size ${driftPosition.size}`) } } catch (err) { console.error(` Error checking ${trade.symbol}:`, err) } } // Check 2: Find trades marked as open but actually closed on Drift // (Less critical but worth detecting) const openTrades = await prisma.trade.findMany({ where: { exitReason: null, }, select: { id: true, positionId: true, symbol: true, createdAt: true, }, }) console.log(` Checking ${openTrades.length} open trades...`) for (const trade of openTrades) { try { const marketIndex = this.getMarketIndex(trade.symbol) if (marketIndex === null) continue const driftPosition = await driftService.getPosition(marketIndex) if (!driftPosition || Math.abs(driftPosition.size) < 0.01) { // MISMATCH: DB says open, Drift says closed const timeSinceExit = Date.now() - new Date(trade.createdAt).getTime() mismatches.push({ tradeId: trade.id, symbol: trade.symbol, expectedState: 'open', actualState: 'closed', driftSize: 0, dbExitReason: null, timeSinceExit, }) console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) console.error(` DB: Open since ${(timeSinceExit / 60000).toFixed(1)}min ago`) console.error(` Drift: Position closed (size 0)`) } } catch (err) { console.error(` Error checking ${trade.symbol}:`, err) } } if (mismatches.length === 0) { console.log(' ✅ No mismatches found - DB and Drift states match') } else { console.error(` ❌ Found ${mismatches.length} mismatches!`) await this.handleMismatches(mismatches) } } catch (error) { console.error('❌ Error running Drift state verification:', error) } return mismatches } /** * Handle detected mismatches */ private async handleMismatches(mismatches: DriftStateMismatch[]): Promise { for (const mismatch of mismatches) { if (mismatch.expectedState === 'closed' && mismatch.actualState === 'open') { // CRITICAL: Position should be closed but is still open on Drift await this.retryClose(mismatch) } else if (mismatch.expectedState === 'open' && mismatch.actualState === 'closed') { // Position closed externally - this is handled by Position Manager's ghost detection console.log(` â„šī¸ ${mismatch.symbol}: Ghost position (will be cleaned by Position Manager)`) } } // Send Telegram alert await this.sendMismatchAlert(mismatches) } /** * Retry closing a position that should be closed but isn't */ private async retryClose(mismatch: DriftStateMismatch): Promise { console.log(`🔄 Retrying close for ${mismatch.symbol}...`) try { const result = await closePosition({ symbol: mismatch.symbol, percentToClose: 100, slippageTolerance: 0.05 // 5% slippage tolerance for market order }) if (result.success) { console.log(` ✅ Successfully closed ${mismatch.symbol}`) console.log(` P&L: $${result.realizedPnL?.toFixed(2) || 0}`) // Update database with retry close info const prisma = getPrismaClient() await prisma.trade.update({ where: { id: mismatch.tradeId }, data: { exitOrderTx: result.transactionSignature || 'RETRY_CLOSE', realizedPnL: result.realizedPnL || 0, configSnapshot: { ...(await prisma.trade.findUnique({ where: { id: mismatch.tradeId }, select: { configSnapshot: true } }))?.configSnapshot as any, retryCloseAttempted: true, retryCloseTime: new Date().toISOString(), } } }) } else { console.error(` ❌ Failed to close ${mismatch.symbol}: ${result.error}`) } } catch (error) { console.error(` ❌ Error retrying close for ${mismatch.symbol}:`, error) } } /** * Send Telegram alert about mismatches */ private async sendMismatchAlert(mismatches: DriftStateMismatch[]): Promise { const criticalMismatches = mismatches.filter(m => m.expectedState === 'closed' && m.actualState === 'open' ) if (criticalMismatches.length === 0) return const message = ` 🚨 DRIFT STATE MISMATCH ALERT Found ${criticalMismatches.length} position(s) that should be closed but are still open on Drift: ${criticalMismatches.map(m => ` 📊 ${m.symbol} DB Status: Closed (${m.dbExitReason}) Drift Status: Open (${m.driftSize.toFixed(2)} tokens) Time since exit: ${(m.timeSinceExit / 60000).toFixed(1)} minutes âš ī¸ Retry close attempted automatically `).join('\n')} This indicates Drift Protocol state propagation issues. Check Drift UI to verify actual position status. `.trim() try { await sendTelegramMessage(message) } catch (error) { console.error('Failed to send Telegram alert:', error) } } /** * Get Drift market index from symbol */ private getMarketIndex(symbol: string): number | null { const marketMap: Record = { 'SOL-PERP': 0, 'BTC-PERP': 1, 'ETH-PERP': 2, } return marketMap[symbol] ?? null } } // Singleton instance let verifierInstance: DriftStateVerifier | null = null export function getDriftStateVerifier(): DriftStateVerifier { if (!verifierInstance) { verifierInstance = new DriftStateVerifier() } return verifierInstance } export function startDriftStateVerifier(): void { const verifier = getDriftStateVerifier() verifier.start() }