From 4ab7bf58da1a4476992bffc1740a5034826e8937 Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Sun, 7 Dec 2025 02:28:10 +0100 Subject: [PATCH] feat: Drift state verifier double-checking system (WIP - build issues) CRITICAL: Position Manager stops monitoring randomly User had to manually close SOL-PERP position after PM stopped at 23:21. Implemented double-checking system to detect when positions marked closed in DB are still open on Drift (and vice versa): 1. DriftStateVerifier service (lib/monitoring/drift-state-verifier.ts) - Runs every 10 minutes automatically - Checks closed trades (24h) vs actual Drift positions - Retries close if mismatch found - Sends Telegram alerts 2. Manual verification API (app/api/monitoring/verify-drift-state) - POST: Force immediate verification check - GET: Service status 3. Integrated into startup (lib/startup/init-position-manager.ts) - Auto-starts on container boot - First check after 2min, then every 10min STATUS: Build failing due to TypeScript compilation timeout Need to fix and deploy, then investigate WHY Position Manager stops. This addresses symptom (stuck positions) but not root cause (PM stopping). --- .../monitoring/verify-drift-state/route.ts | 97 ++++++ lib/monitoring/drift-state-verifier.ts | 319 ++++++++++++++++++ lib/startup/init-position-manager.ts | 5 + 3 files changed, 421 insertions(+) create mode 100644 app/api/monitoring/verify-drift-state/route.ts create mode 100644 lib/monitoring/drift-state-verifier.ts diff --git a/app/api/monitoring/verify-drift-state/route.ts b/app/api/monitoring/verify-drift-state/route.ts new file mode 100644 index 0000000..290db38 --- /dev/null +++ b/app/api/monitoring/verify-drift-state/route.ts @@ -0,0 +1,97 @@ +/** + * Force Drift State Check API Endpoint + * + * Manually trigger Drift state verification and retry closing + * any positions that should be closed but aren't. + * + * POST /api/monitoring/verify-drift-state + * Authorization: Bearer + */ + +import { NextRequest, NextResponse } from 'next/server' +import { getDriftStateVerifier } from '@/lib/monitoring/drift-state-verifier' + +export async function POST(request: NextRequest) { + try { + // Verify authorization + const authHeader = request.headers.get('authorization') + const expectedAuth = `Bearer ${process.env.API_SECRET_KEY}` + + if (!authHeader || authHeader !== expectedAuth) { + return NextResponse.json( + { success: false, error: 'Unauthorized' }, + { status: 401 } + ) + } + + console.log('🔍 Manual Drift state verification requested...') + + const verifier = getDriftStateVerifier() + const mismatches = await verifier.runVerification() + + return NextResponse.json({ + success: true, + timestamp: new Date().toISOString(), + mismatchesFound: mismatches.length, + mismatches: mismatches.map(m => ({ + tradeId: m.tradeId, + symbol: m.symbol, + expectedState: m.expectedState, + actualState: m.actualState, + driftSize: m.driftSize, + dbExitReason: m.dbExitReason, + timeSinceExit: m.timeSinceExit, + })), + message: mismatches.length === 0 + ? 'All positions match between database and Drift' + : `Found ${mismatches.length} mismatches - retry close attempted for critical cases` + }) + + } catch (error) { + console.error('❌ Error in Drift state verification:', error) + + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }, + { status: 500 } + ) + } +} + +/** + * Get current verification service status + * GET /api/monitoring/verify-drift-state + */ +export async function GET(request: NextRequest) { + try { + const authHeader = request.headers.get('authorization') + const expectedAuth = `Bearer ${process.env.API_SECRET_KEY}` + + if (!authHeader || authHeader !== expectedAuth) { + return NextResponse.json( + { success: false, error: 'Unauthorized' }, + { status: 401 } + ) + } + + return NextResponse.json({ + success: true, + service: 'Drift State Verifier', + status: 'running', + checkInterval: '10 minutes', + description: 'Automatically verifies closed positions are actually closed on Drift. Retries close if mismatches found.', + endpoints: { + manualCheck: 'POST /api/monitoring/verify-drift-state', + status: 'GET /api/monitoring/verify-drift-state' + } + }) + + } catch (error) { + return NextResponse.json( + { success: false, error: 'Internal error' }, + { status: 500 } + ) + } +} diff --git a/lib/monitoring/drift-state-verifier.ts b/lib/monitoring/drift-state-verifier.ts new file mode 100644 index 0000000..4580589 --- /dev/null +++ b/lib/monitoring/drift-state-verifier.ts @@ -0,0 +1,319 @@ +/** + * Drift State Verifier Service + * + * Double-checks that positions marked as closed in our database + * are actually closed on Drift Protocol. If mismatches found, + * attempts to close the position again. + * + * Background: Drift occasionally confirms close transactions but + * doesn't actually close the position (state propagation delay or + * partial fill issues). This service detects and fixes those cases. + * + * Created: Dec 7, 2025 + */ + +import { getDriftService } from '../drift/client' +import { getPrismaClient } from '../database/trades' +import { closePosition } from '../drift/orders' +import { sendTelegramMessage } from '../notifications/telegram' + +export interface DriftStateMismatch { + tradeId: string + symbol: string + expectedState: 'closed' | 'open' + actualState: 'closed' | 'open' + driftSize: number + dbExitReason: string | null + timeSinceExit: number // milliseconds +} + +class DriftStateVerifier { + private isRunning: boolean = false + private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes + private intervalId: NodeJS.Timeout | null = null + + /** + * Start the periodic verification service + */ + start(): void { + if (this.isRunning) { + console.log('🔍 Drift state verifier already running') + return + } + + console.log('🔍 Starting Drift state verifier (checks every 10 minutes)') + this.isRunning = true + + // Run first check after 2 minutes (allow time for initial startup) + setTimeout(() => { + this.runVerification().catch(err => { + console.error('❌ Error in initial Drift state verification:', err) + }) + }, 2 * 60 * 1000) + + // Then run every 10 minutes + this.intervalId = setInterval(() => { + this.runVerification().catch(err => { + console.error('❌ Error in Drift state verification:', err) + }) + }, this.checkIntervalMs) + } + + /** + * Stop the periodic verification service + */ + stop(): void { + if (this.intervalId) { + clearInterval(this.intervalId) + this.intervalId = null + } + this.isRunning = false + console.log('🔍 Drift state verifier stopped') + } + + /** + * Run verification check once (can be called manually) + */ + async runVerification(): Promise { + console.log('🔍 Running Drift state verification...') + + const mismatches: DriftStateMismatch[] = [] + + try { + const driftService = await getDriftService() + const prisma = getPrismaClient() + + // Check 1: Find trades marked as closed in last 24 hours + // These should definitely not exist on Drift anymore + const recentlyClosedTrades = await prisma.trade.findMany({ + where: { + exitReason: { not: null }, + exitTime: { + gte: new Date(Date.now() - 24 * 60 * 60 * 1000) + } + }, + select: { + id: true, + positionId: true, + symbol: true, + exitReason: true, + exitTime: true, + }, + }) + + console.log(` Checking ${recentlyClosedTrades.length} recently closed trades...`) + + for (const trade of recentlyClosedTrades) { + try { + // Extract market index from symbol (SOL-PERP → 0, ETH-PERP → 1, etc.) + const marketIndex = this.getMarketIndex(trade.symbol) + if (marketIndex === null) continue + + // Query Drift for position + const driftPosition = await driftService.getPosition(marketIndex) + + if (driftPosition && Math.abs(driftPosition.size) >= 0.01) { + // MISMATCH: DB says closed, Drift says open + const timeSinceExit = Date.now() - new Date(trade.exitTime!).getTime() + + mismatches.push({ + tradeId: trade.id, + symbol: trade.symbol, + expectedState: 'closed', + actualState: 'open', + driftSize: Math.abs(driftPosition.size), + dbExitReason: trade.exitReason, + timeSinceExit, + }) + + console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) + console.error(` DB: Closed ${(timeSinceExit / 60000).toFixed(1)}min ago (${trade.exitReason})`) + console.error(` Drift: Still open with size ${driftPosition.size}`) + } + } catch (err) { + console.error(` Error checking ${trade.symbol}:`, err) + } + } + + // Check 2: Find trades marked as open but actually closed on Drift + // (Less critical but worth detecting) + const openTrades = await prisma.trade.findMany({ + where: { + exitReason: null, + }, + select: { + id: true, + positionId: true, + symbol: true, + createdAt: true, + }, + }) + + console.log(` Checking ${openTrades.length} open trades...`) + + for (const trade of openTrades) { + try { + const marketIndex = this.getMarketIndex(trade.symbol) + if (marketIndex === null) continue + + const driftPosition = await driftService.getPosition(marketIndex) + + if (!driftPosition || Math.abs(driftPosition.size) < 0.01) { + // MISMATCH: DB says open, Drift says closed + const timeSinceExit = Date.now() - new Date(trade.createdAt).getTime() + + mismatches.push({ + tradeId: trade.id, + symbol: trade.symbol, + expectedState: 'open', + actualState: 'closed', + driftSize: 0, + dbExitReason: null, + timeSinceExit, + }) + + console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`) + console.error(` DB: Open since ${(timeSinceExit / 60000).toFixed(1)}min ago`) + console.error(` Drift: Position closed (size 0)`) + } + } catch (err) { + console.error(` Error checking ${trade.symbol}:`, err) + } + } + + if (mismatches.length === 0) { + console.log(' ✅ No mismatches found - DB and Drift states match') + } else { + console.error(` ❌ Found ${mismatches.length} mismatches!`) + await this.handleMismatches(mismatches) + } + + } catch (error) { + console.error('❌ Error running Drift state verification:', error) + } + + return mismatches + } + + /** + * Handle detected mismatches + */ + private async handleMismatches(mismatches: DriftStateMismatch[]): Promise { + for (const mismatch of mismatches) { + if (mismatch.expectedState === 'closed' && mismatch.actualState === 'open') { + // CRITICAL: Position should be closed but is still open on Drift + await this.retryClose(mismatch) + } else if (mismatch.expectedState === 'open' && mismatch.actualState === 'closed') { + // Position closed externally - this is handled by Position Manager's ghost detection + console.log(` ℹī¸ ${mismatch.symbol}: Ghost position (will be cleaned by Position Manager)`) + } + } + + // Send Telegram alert + await this.sendMismatchAlert(mismatches) + } + + /** + * Retry closing a position that should be closed but isn't + */ + private async retryClose(mismatch: DriftStateMismatch): Promise { + console.log(`🔄 Retrying close for ${mismatch.symbol}...`) + + try { + const result = await closePosition({ + symbol: mismatch.symbol, + percentToClose: 100, + slippageTolerance: 0.05 // 5% slippage tolerance for market order + }) + + if (result.success) { + console.log(` ✅ Successfully closed ${mismatch.symbol}`) + console.log(` P&L: $${result.realizedPnL?.toFixed(2) || 0}`) + + // Update database with retry close info + const prisma = getPrismaClient() + await prisma.trade.update({ + where: { id: mismatch.tradeId }, + data: { + exitOrderTx: result.transactionSignature || 'RETRY_CLOSE', + realizedPnL: result.realizedPnL || 0, + configSnapshot: { + ...(await prisma.trade.findUnique({ + where: { id: mismatch.tradeId }, + select: { configSnapshot: true } + }))?.configSnapshot as any, + retryCloseAttempted: true, + retryCloseTime: new Date().toISOString(), + } + } + }) + } else { + console.error(` ❌ Failed to close ${mismatch.symbol}: ${result.error}`) + } + } catch (error) { + console.error(` ❌ Error retrying close for ${mismatch.symbol}:`, error) + } + } + + /** + * Send Telegram alert about mismatches + */ + private async sendMismatchAlert(mismatches: DriftStateMismatch[]): Promise { + const criticalMismatches = mismatches.filter(m => + m.expectedState === 'closed' && m.actualState === 'open' + ) + + if (criticalMismatches.length === 0) return + + const message = ` +🚨 DRIFT STATE MISMATCH ALERT + +Found ${criticalMismatches.length} position(s) that should be closed but are still open on Drift: + +${criticalMismatches.map(m => ` +📊 ${m.symbol} + DB Status: Closed (${m.dbExitReason}) + Drift Status: Open (${m.driftSize.toFixed(2)} tokens) + Time since exit: ${(m.timeSinceExit / 60000).toFixed(1)} minutes + + ⚠ī¸ Retry close attempted automatically +`).join('\n')} + +This indicates Drift Protocol state propagation issues. +Check Drift UI to verify actual position status. +`.trim() + + try { + await sendTelegramMessage(message) + } catch (error) { + console.error('Failed to send Telegram alert:', error) + } + } + + /** + * Get Drift market index from symbol + */ + private getMarketIndex(symbol: string): number | null { + const marketMap: Record = { + 'SOL-PERP': 0, + 'BTC-PERP': 1, + 'ETH-PERP': 2, + } + return marketMap[symbol] ?? null + } +} + +// Singleton instance +let verifierInstance: DriftStateVerifier | null = null + +export function getDriftStateVerifier(): DriftStateVerifier { + if (!verifierInstance) { + verifierInstance = new DriftStateVerifier() + } + return verifierInstance +} + +export function startDriftStateVerifier(): void { + const verifier = getDriftStateVerifier() + verifier.start() +} diff --git a/lib/startup/init-position-manager.ts b/lib/startup/init-position-manager.ts index 2243c3f..5487255 100644 --- a/lib/startup/init-position-manager.ts +++ b/lib/startup/init-position-manager.ts @@ -14,6 +14,7 @@ import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker' import { startStopHuntTracking } from '../trading/stop-hunt-tracker' import { startSmartValidation } from '../trading/smart-validation-queue' import { startDataCleanup } from '../maintenance/data-cleanup' +import { startDriftStateVerifier } from '../monitoring/drift-state-verifier' import { logCriticalError } from '../utils/persistent-logger' import { sendPositionClosedNotification } from '../notifications/telegram' @@ -51,6 +52,10 @@ export async function initializePositionManagerOnStartup() { console.log('🧠 Starting smart entry validation system...') await startSmartValidation() + // Start Drift state verifier (Dec 7, 2025) + console.log('🔍 Starting Drift state verifier (double-checks closed positions every 10 min)...') + startDriftStateVerifier() + // CRITICAL: Run database sync validator to clean up duplicates const { validateAllOpenTrades } = await import('../database/sync-validator') console.log('🔍 Running database sync validation before Position Manager init...')