feat: Drift state verifier double-checking system (WIP - build issues)

CRITICAL: Position Manager stops monitoring randomly
User had to manually close SOL-PERP position after PM stopped at 23:21.

Implemented double-checking system to detect when positions marked
closed in DB are still open on Drift (and vice versa):

1. DriftStateVerifier service (lib/monitoring/drift-state-verifier.ts)
   - Runs every 10 minutes automatically
   - Checks closed trades (24h) vs actual Drift positions
   - Retries close if mismatch found
   - Sends Telegram alerts

2. Manual verification API (app/api/monitoring/verify-drift-state)
   - POST: Force immediate verification check
   - GET: Service status

3. Integrated into startup (lib/startup/init-position-manager.ts)
   - Auto-starts on container boot
   - First check after 2min, then every 10min

STATUS: Build failing due to TypeScript compilation timeout
Need to fix and deploy, then investigate WHY Position Manager stops.

This addresses symptom (stuck positions) but not root cause (PM stopping).

app/api/monitoring/verify-drift-state/route.ts

@@ -0,0 +1,97 @@
+/**
+ * Force Drift State Check API Endpoint
+ * 
+ * Manually trigger Drift state verification and retry closing
+ * any positions that should be closed but aren't.
+ * 
+ * POST /api/monitoring/verify-drift-state
+ * Authorization: Bearer <API_SECRET_KEY>
+ */
+
+import { NextRequest, NextResponse } from 'next/server'
+import { getDriftStateVerifier } from '@/lib/monitoring/drift-state-verifier'
+
+export async function POST(request: NextRequest) {
+  try {
+    // Verify authorization
+    const authHeader = request.headers.get('authorization')
+    const expectedAuth = `Bearer ${process.env.API_SECRET_KEY}`
+    
+    if (!authHeader || authHeader !== expectedAuth) {
+      return NextResponse.json(
+        { success: false, error: 'Unauthorized' },
+        { status: 401 }
+      )
+    }
+
+    console.log('🔍 Manual Drift state verification requested...')
+
+    const verifier = getDriftStateVerifier()
+    const mismatches = await verifier.runVerification()
+
+    return NextResponse.json({
+      success: true,
+      timestamp: new Date().toISOString(),
+      mismatchesFound: mismatches.length,
+      mismatches: mismatches.map(m => ({
+        tradeId: m.tradeId,
+        symbol: m.symbol,
+        expectedState: m.expectedState,
+        actualState: m.actualState,
+        driftSize: m.driftSize,
+        dbExitReason: m.dbExitReason,
+        timeSinceExit: m.timeSinceExit,
+      })),
+      message: mismatches.length === 0
+        ? 'All positions match between database and Drift'
+        : `Found ${mismatches.length} mismatches - retry close attempted for critical cases`
+    })
+
+  } catch (error) {
+    console.error('❌ Error in Drift state verification:', error)
+    
+    return NextResponse.json(
+      {
+        success: false,
+        error: error instanceof Error ? error.message : 'Unknown error'
+      },
+      { status: 500 }
+    )
+  }
+}
+
+/**
+ * Get current verification service status
+ * GET /api/monitoring/verify-drift-state
+ */
+export async function GET(request: NextRequest) {
+  try {
+    const authHeader = request.headers.get('authorization')
+    const expectedAuth = `Bearer ${process.env.API_SECRET_KEY}`
+    
+    if (!authHeader || authHeader !== expectedAuth) {
+      return NextResponse.json(
+        { success: false, error: 'Unauthorized' },
+        { status: 401 }
+      )
+    }
+
+    return NextResponse.json({
+      success: true,
+      service: 'Drift State Verifier',
+      status: 'running',
+      checkInterval: '10 minutes',
+      description: 'Automatically verifies closed positions are actually closed on Drift. Retries close if mismatches found.',
+      endpoints: {
+        manualCheck: 'POST /api/monitoring/verify-drift-state',
+        status: 'GET /api/monitoring/verify-drift-state'
+      }
+    })
+
+  } catch (error) {
+    return NextResponse.json(
+      { success: false, error: 'Internal error' },
+      { status: 500 }
+    )
+  }
+}

lib/monitoring/drift-state-verifier.ts

@@ -0,0 +1,319 @@
+/**
+ * Drift State Verifier Service
+ * 
+ * Double-checks that positions marked as closed in our database
+ * are actually closed on Drift Protocol. If mismatches found,
+ * attempts to close the position again.
+ * 
+ * Background: Drift occasionally confirms close transactions but
+ * doesn't actually close the position (state propagation delay or
+ * partial fill issues). This service detects and fixes those cases.
+ * 
+ * Created: Dec 7, 2025
+ */
+
+import { getDriftService } from '../drift/client'
+import { getPrismaClient } from '../database/trades'
+import { closePosition } from '../drift/orders'
+import { sendTelegramMessage } from '../notifications/telegram'
+
+export interface DriftStateMismatch {
+  tradeId: string
+  symbol: string
+  expectedState: 'closed' | 'open'
+  actualState: 'closed' | 'open'
+  driftSize: number
+  dbExitReason: string | null
+  timeSinceExit: number // milliseconds
+}
+
+class DriftStateVerifier {
+  private isRunning: boolean = false
+  private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes
+  private intervalId: NodeJS.Timeout | null = null
+
+  /**
+   * Start the periodic verification service
+   */
+  start(): void {
+    if (this.isRunning) {
+      console.log('🔍 Drift state verifier already running')
+      return
+    }
+
+    console.log('🔍 Starting Drift state verifier (checks every 10 minutes)')
+    this.isRunning = true
+
+    // Run first check after 2 minutes (allow time for initial startup)
+    setTimeout(() => {
+      this.runVerification().catch(err => {
+        console.error('❌ Error in initial Drift state verification:', err)
+      })
+    }, 2 * 60 * 1000)
+
+    // Then run every 10 minutes
+    this.intervalId = setInterval(() => {
+      this.runVerification().catch(err => {
+        console.error('❌ Error in Drift state verification:', err)
+      })
+    }, this.checkIntervalMs)
+  }
+
+  /**
+   * Stop the periodic verification service
+   */
+  stop(): void {
+    if (this.intervalId) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+    }
+    this.isRunning = false
+    console.log('🔍 Drift state verifier stopped')
+  }
+
+  /**
+   * Run verification check once (can be called manually)
+   */
+  async runVerification(): Promise<DriftStateMismatch[]> {
+    console.log('🔍 Running Drift state verification...')
+
+    const mismatches: DriftStateMismatch[] = []
+
+    try {
+      const driftService = await getDriftService()
+      const prisma = getPrismaClient()
+
+      // Check 1: Find trades marked as closed in last 24 hours
+      // These should definitely not exist on Drift anymore
+      const recentlyClosedTrades = await prisma.trade.findMany({
+        where: {
+          exitReason: { not: null },
+          exitTime: {
+            gte: new Date(Date.now() - 24 * 60 * 60 * 1000)
+          }
+        },
+        select: {
+          id: true,
+          positionId: true,
+          symbol: true,
+          exitReason: true,
+          exitTime: true,
+        },
+      })
+
+      console.log(`   Checking ${recentlyClosedTrades.length} recently closed trades...`)
+
+      for (const trade of recentlyClosedTrades) {
+        try {
+          // Extract market index from symbol (SOL-PERP → 0, ETH-PERP → 1, etc.)
+          const marketIndex = this.getMarketIndex(trade.symbol)
+          if (marketIndex === null) continue
+
+          // Query Drift for position
+          const driftPosition = await driftService.getPosition(marketIndex)
+          
+          if (driftPosition && Math.abs(driftPosition.size) >= 0.01) {
+            // MISMATCH: DB says closed, Drift says open
+            const timeSinceExit = Date.now() - new Date(trade.exitTime!).getTime()
+            
+            mismatches.push({
+              tradeId: trade.id,
+              symbol: trade.symbol,
+              expectedState: 'closed',
+              actualState: 'open',
+              driftSize: Math.abs(driftPosition.size),
+              dbExitReason: trade.exitReason,
+              timeSinceExit,
+            })
+
+            console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`)
+            console.error(`   DB: Closed ${(timeSinceExit / 60000).toFixed(1)}min ago (${trade.exitReason})`)
+            console.error(`   Drift: Still open with size ${driftPosition.size}`)
+          }
+        } catch (err) {
+          console.error(`   Error checking ${trade.symbol}:`, err)
+        }
+      }
+
+      // Check 2: Find trades marked as open but actually closed on Drift
+      // (Less critical but worth detecting)
+      const openTrades = await prisma.trade.findMany({
+        where: {
+          exitReason: null,
+        },
+        select: {
+          id: true,
+          positionId: true,
+          symbol: true,
+          createdAt: true,
+        },
+      })
+
+      console.log(`   Checking ${openTrades.length} open trades...`)
+
+      for (const trade of openTrades) {
+        try {
+          const marketIndex = this.getMarketIndex(trade.symbol)
+          if (marketIndex === null) continue
+
+          const driftPosition = await driftService.getPosition(marketIndex)
+          
+          if (!driftPosition || Math.abs(driftPosition.size) < 0.01) {
+            // MISMATCH: DB says open, Drift says closed
+            const timeSinceExit = Date.now() - new Date(trade.createdAt).getTime()
+            
+            mismatches.push({
+              tradeId: trade.id,
+              symbol: trade.symbol,
+              expectedState: 'open',
+              actualState: 'closed',
+              driftSize: 0,
+              dbExitReason: null,
+              timeSinceExit,
+            })
+
+            console.error(`🚨 MISMATCH DETECTED: ${trade.symbol}`)
+            console.error(`   DB: Open since ${(timeSinceExit / 60000).toFixed(1)}min ago`)
+            console.error(`   Drift: Position closed (size 0)`)
+          }
+        } catch (err) {
+          console.error(`   Error checking ${trade.symbol}:`, err)
+        }
+      }
+
+      if (mismatches.length === 0) {
+        console.log('   ✅ No mismatches found - DB and Drift states match')
+      } else {
+        console.error(`   ❌ Found ${mismatches.length} mismatches!`)
+        await this.handleMismatches(mismatches)
+      }
+
+    } catch (error) {
+      console.error('❌ Error running Drift state verification:', error)
+    }
+
+    return mismatches
+  }
+
+  /**
+   * Handle detected mismatches
+   */
+  private async handleMismatches(mismatches: DriftStateMismatch[]): Promise<void> {
+    for (const mismatch of mismatches) {
+      if (mismatch.expectedState === 'closed' && mismatch.actualState === 'open') {
+        // CRITICAL: Position should be closed but is still open on Drift
+        await this.retryClose(mismatch)
+      } else if (mismatch.expectedState === 'open' && mismatch.actualState === 'closed') {
+        // Position closed externally - this is handled by Position Manager's ghost detection
+        console.log(`   ℹ️ ${mismatch.symbol}: Ghost position (will be cleaned by Position Manager)`)
+      }
+    }
+
+    // Send Telegram alert
+    await this.sendMismatchAlert(mismatches)
+  }
+
+  /**
+   * Retry closing a position that should be closed but isn't
+   */
+  private async retryClose(mismatch: DriftStateMismatch): Promise<void> {
+    console.log(`🔄 Retrying close for ${mismatch.symbol}...`)
+
+    try {
+      const result = await closePosition({
+        symbol: mismatch.symbol,
+        percentToClose: 100,
+        slippageTolerance: 0.05 // 5% slippage tolerance for market order
+      })
+
+      if (result.success) {
+        console.log(`   ✅ Successfully closed ${mismatch.symbol}`)
+        console.log(`   P&L: $${result.realizedPnL?.toFixed(2) || 0}`)
+        
+        // Update database with retry close info
+        const prisma = getPrismaClient()
+        await prisma.trade.update({
+          where: { id: mismatch.tradeId },
+          data: {
+            exitOrderTx: result.transactionSignature || 'RETRY_CLOSE',
+            realizedPnL: result.realizedPnL || 0,
+            configSnapshot: {
+              ...(await prisma.trade.findUnique({
+                where: { id: mismatch.tradeId },
+                select: { configSnapshot: true }
+              }))?.configSnapshot as any,
+              retryCloseAttempted: true,
+              retryCloseTime: new Date().toISOString(),
+            }
+          }
+        })
+      } else {
+        console.error(`   ❌ Failed to close ${mismatch.symbol}: ${result.error}`)
+      }
+    } catch (error) {
+      console.error(`   ❌ Error retrying close for ${mismatch.symbol}:`, error)
+    }
+  }
+
+  /**
+   * Send Telegram alert about mismatches
+   */
+  private async sendMismatchAlert(mismatches: DriftStateMismatch[]): Promise<void> {
+    const criticalMismatches = mismatches.filter(m => 
+      m.expectedState === 'closed' && m.actualState === 'open'
+    )
+
+    if (criticalMismatches.length === 0) return
+
+    const message = `
+🚨 DRIFT STATE MISMATCH ALERT
+
+Found ${criticalMismatches.length} position(s) that should be closed but are still open on Drift:
+
+${criticalMismatches.map(m => `
+📊 ${m.symbol}
+   DB Status: Closed (${m.dbExitReason})
+   Drift Status: Open (${m.driftSize.toFixed(2)} tokens)
+   Time since exit: ${(m.timeSinceExit / 60000).toFixed(1)} minutes
+   
+   ⚠️ Retry close attempted automatically
+`).join('\n')}
+
+This indicates Drift Protocol state propagation issues.
+Check Drift UI to verify actual position status.
+`.trim()
+
+    try {
+      await sendTelegramMessage(message)
+    } catch (error) {
+      console.error('Failed to send Telegram alert:', error)
+    }
+  }
+
+  /**
+   * Get Drift market index from symbol
+   */
+  private getMarketIndex(symbol: string): number | null {
+    const marketMap: Record<string, number> = {
+      'SOL-PERP': 0,
+      'BTC-PERP': 1,
+      'ETH-PERP': 2,
+    }
+    return marketMap[symbol] ?? null
+  }
+}
+
+// Singleton instance
+let verifierInstance: DriftStateVerifier | null = null
+
+export function getDriftStateVerifier(): DriftStateVerifier {
+  if (!verifierInstance) {
+    verifierInstance = new DriftStateVerifier()
+  }
+  return verifierInstance
+}
+
+export function startDriftStateVerifier(): void {
+  const verifier = getDriftStateVerifier()
+  verifier.start()
+}

lib/startup/init-position-manager.ts

@@ -14,6 +14,7 @@ import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker'
 import { startStopHuntTracking } from '../trading/stop-hunt-tracker'
 import { startSmartValidation } from '../trading/smart-validation-queue'
 import { startDataCleanup } from '../maintenance/data-cleanup'
+import { startDriftStateVerifier } from '../monitoring/drift-state-verifier'
 import { logCriticalError } from '../utils/persistent-logger'
 import { sendPositionClosedNotification } from '../notifications/telegram'
 
@@ -51,6 +52,10 @@ export async function initializePositionManagerOnStartup() {
     console.log('🧠 Starting smart entry validation system...')
     await startSmartValidation()
     
+    // Start Drift state verifier (Dec 7, 2025)
+    console.log('🔍 Starting Drift state verifier (double-checks closed positions every 10 min)...')
+    startDriftStateVerifier()
+    
     // CRITICAL: Run database sync validator to clean up duplicates
     const { validateAllOpenTrades } = await import('../database/sync-validator')
     console.log('🔍 Running database sync validation before Position Manager init...')