/** * Position Manager Health Check * * CRITICAL: Verifies Position Manager is actually monitoring positions * * Bug History: * - $1,000+ losses because Position Manager logged "added" but never monitored * - Silent SL placement failures left positions unprotected * - Orphan detection cancelled orders on active positions * * This health check runs every 30 seconds to detect these critical failures. * * Created: Dec 8, 2025 */ import { getInitializedPositionManager } from '../trading/position-manager' import { getOpenTrades, getPrismaClient } from '../database/trades' import { getDriftService } from '../drift/client' import { getMergedConfig } from '../../config/trading' export interface HealthCheckResult { isHealthy: boolean issues: string[] warnings: string[] info: { dbOpenTrades: number pmActiveTrades: number pmMonitoring: boolean driftPositions: number unprotectedPositions: number } autoSyncTriggered?: boolean } // Cooldown to prevent sync spam (5 minutes) let lastAutoSyncTime = 0 const AUTO_SYNC_COOLDOWN_MS = 5 * 60 * 1000 /** * Automatically trigger position sync when untracked positions detected * * CRITICAL: This prevents the $1,000 loss scenario where Telegram bot * opens positions but Position Manager never tracks them. * * Cooldown: 5 minutes between auto-syncs to prevent spam */ async function autoSyncUntrackedPositions(): Promise { const now = Date.now() // Check cooldown if (now - lastAutoSyncTime < AUTO_SYNC_COOLDOWN_MS) { const remainingSeconds = Math.ceil((AUTO_SYNC_COOLDOWN_MS - (now - lastAutoSyncTime)) / 1000) console.log(`ā³ Auto-sync cooldown active (${remainingSeconds}s remaining)`) return false } try { console.log('šŸ”„ AUTO-SYNC TRIGGERED: Untracked positions detected, syncing with Drift...') // Call the sync endpoint internally const pm = await getInitializedPositionManager() const driftService = getDriftService() const driftPositions = await driftService.getAllPositions() console.log(`šŸ“Š Found ${driftPositions.length} positions on Drift`) let added = 0 for (const driftPos of driftPositions) { const isTracked = Array.from((pm as any).activeTrades.values()).some( (t: any) => t.symbol === driftPos.symbol ) if (!isTracked && Math.abs(driftPos.size) > 0) { console.log(`āž• Auto-adding ${driftPos.symbol} to Position Manager`) // Import sync logic const { syncSinglePosition } = await import('../trading/sync-helper') await syncSinglePosition(driftPos, pm) added++ } } lastAutoSyncTime = now console.log(`āœ… AUTO-SYNC COMPLETE: Added ${added} position(s) to monitoring`) return true } catch (error) { console.error('āŒ Auto-sync failed:', error) return false } } function calculatePrice(entry: number, percent: number, direction: 'long' | 'short'): number { return direction === 'long' ? entry * (1 + percent / 100) : entry * (1 - percent / 100) } async function ensureExitOrdersForTrade( trade: any, config: ReturnType ): Promise<{ placed: boolean; message?: string; error?: string }> { try { const positionSizeUSD = trade.positionSizeUSD || trade.positionSize || 0 if (!positionSizeUSD || !trade.entryPrice) { return { placed: false, error: 'Missing position size or entry price' } } const direction: 'long' | 'short' = trade.direction === 'short' ? 'short' : 'long' const tp1Price = trade.takeProfit1Price || calculatePrice(trade.entryPrice, config.takeProfit1Percent, direction) const tp2Price = trade.takeProfit2Price || calculatePrice(trade.entryPrice, config.takeProfit2Percent, direction) const stopLossPrice = trade.stopLossPrice || calculatePrice(trade.entryPrice, config.stopLossPercent, direction) const tp1SizePercent = trade.tp1SizePercent ?? config.takeProfit1SizePercent const tp2SizePercentRaw = trade.tp2SizePercent ?? config.takeProfit2SizePercent ?? 0 const tp2SizePercent = config.useTp2AsTriggerOnly && tp2SizePercentRaw <= 0 ? 0 : tp2SizePercentRaw const softStopPrice = config.useDualStops ? calculatePrice(trade.entryPrice, config.softStopPercent, direction) : undefined const hardStopPrice = config.useDualStops ? calculatePrice(trade.entryPrice, config.hardStopPercent, direction) : undefined const { placeExitOrders } = await import('../drift/orders') const placeResult = await placeExitOrders({ symbol: trade.symbol, positionSizeUSD, entryPrice: trade.entryPrice, tp1Price, tp2Price, stopLossPrice, tp1SizePercent, tp2SizePercent, direction, useDualStops: config.useDualStops, softStopPrice, softStopBuffer: config.softStopBuffer, hardStopPrice, }) if (!placeResult.success) { return { placed: false, error: placeResult.error || 'Unknown error placing exit orders' } } const signatures = placeResult.signatures || [] const normalizedTp2Percent = tp2SizePercent === undefined ? 100 : Math.max(0, tp2SizePercent) const tp1USD = (positionSizeUSD * tp1SizePercent) / 100 const remainingAfterTP1 = positionSizeUSD - tp1USD const tp2USD = (remainingAfterTP1 * normalizedTp2Percent) / 100 let idx = 0 const updateData: any = {} if (tp1USD > 0 && idx < signatures.length) { updateData.tp1OrderTx = signatures[idx++] } if (normalizedTp2Percent > 0 && idx < signatures.length) { updateData.tp2OrderTx = signatures[idx++] } if (config.useDualStops && softStopPrice && hardStopPrice) { if (idx < signatures.length) { updateData.softStopOrderTx = signatures[idx++] } if (idx < signatures.length) { updateData.hardStopOrderTx = signatures[idx++] } } else if (idx < signatures.length) { updateData.slOrderTx = signatures[idx++] } const prisma = getPrismaClient() await prisma.trade.update({ where: { id: trade.id }, data: updateData }) return { placed: true, message: 'Protective exits placed' } } catch (error) { return { placed: false, error: error instanceof Error ? error.message : String(error) } } } /** * Check Position Manager health * * CRITICAL CHECKS: * 1. If DB has open trades, Position Manager MUST be monitoring * 2. If Position Manager has trades, monitoring MUST be active * 3. All open positions MUST have TP/SL orders on-chain * 4. Position Manager trade count MUST match Drift position count */ export async function checkPositionManagerHealth(): Promise { const issues: string[] = [] const warnings: string[] = [] const config = getMergedConfig() try { // Get database open trades const dbTrades = await getOpenTrades() const dbOpenCount = dbTrades.length // Get Position Manager state const pm = await getInitializedPositionManager() const pmState = (pm as any) let pmActiveTrades = pmState.activeTrades?.size || 0 let pmMonitoring = pmState.isMonitoring || false // Get Drift positions const driftService = getDriftService() const positions = await driftService.getAllPositions() const driftPositions = positions.filter(p => Math.abs(p.size) > 0).length // CRITICAL CHECK #1: DB has open trades but PM not monitoring if (dbOpenCount > 0 && !pmMonitoring) { console.log('šŸ› ļø Health monitor: Attempting automatic monitoring restore from DB...') try { await pm.initialize(true) pmActiveTrades = (pm as any).activeTrades?.size || 0 pmMonitoring = (pm as any).isMonitoring || false } catch (restoreError) { console.error('āŒ Failed to auto-restore monitoring:', restoreError) } } // Re-check after attempted restore if (dbOpenCount > 0 && !pmMonitoring) { issues.push(`āŒ CRITICAL: ${dbOpenCount} open trades in DB but Position Manager NOT monitoring!`) issues.push(` This means NO TP/SL protection, NO monitoring, UNCONTROLLED RISK`) issues.push(` ACTION REQUIRED: Restart container to restore monitoring`) } // CRITICAL CHECK #2: PM has trades but not monitoring if (pmActiveTrades > 0 && !pmMonitoring) { issues.push(`āŒ CRITICAL: Position Manager has ${pmActiveTrades} active trades but monitoring is OFF!`) issues.push(` This is the $1,000 loss bug - trades "added" but never monitored`) issues.push(` ACTION REQUIRED: Fix startMonitoring() function`) } // CRITICAL CHECK #3: DB vs PM mismatch if (dbOpenCount !== pmActiveTrades) { warnings.push(`āš ļø WARNING: DB has ${dbOpenCount} open trades, PM has ${pmActiveTrades} active trades`) warnings.push(` Possible orphaned position or monitoring not started`) } // CRITICAL CHECK #4: PM vs Drift mismatch if (pmActiveTrades !== driftPositions) { warnings.push(`āš ļø WARNING: Position Manager has ${pmActiveTrades} trades, Drift has ${driftPositions} positions`) warnings.push(` Possible untracked position or external closure`) // AUTO-SYNC: If Drift has MORE positions than PM, sync automatically if (driftPositions > pmActiveTrades) { console.log(`🚨 UNTRACKED POSITIONS DETECTED: Drift has ${driftPositions}, PM has ${pmActiveTrades}`) const synced = await autoSyncUntrackedPositions() if (synced) { warnings.push(`āœ… AUTO-SYNC: Triggered position sync to restore protection`) // Re-check PM state after sync pmActiveTrades = (pm as any).activeTrades?.size || 0 } } } // Check for unprotected positions let unprotectedPositions = 0 for (const trade of dbTrades) { const hasDbSignatures = !!(trade.slOrderTx || trade.softStopOrderTx || trade.hardStopOrderTx) if (!hasDbSignatures) { unprotectedPositions++ issues.push(`āŒ CRITICAL: Position ${trade.symbol} (${trade.id}) has NO STOP LOSS ORDERS!`) issues.push(` Entry: $${trade.entryPrice}, Size: $${trade.positionSizeUSD}`) issues.push(` Attempting automatic protective order placement...`) const remediation = await ensureExitOrdersForTrade(trade, config) if (remediation.placed) { warnings.push(`āœ… Auto-placed protective exit orders for ${trade.symbol} (${trade.id})`) } else if (remediation.error) { issues.push(` āŒ Failed to auto-place exits: ${remediation.error}`) } } if (!trade.tp1OrderTx) { warnings.push(`āš ļø Position ${trade.symbol} missing TP1 order (not synced)`) } if (!trade.tp2OrderTx) { warnings.push(`āš ļø Position ${trade.symbol} missing TP2 order (not synced)`) } } // BUG #89 FIX PART 3 (Dec 16, 2025): Check for fractional position remnants // Query database for positions marked as FRACTIONAL_REMNANT in last 24 hours // These require manual intervention via Drift UI const prisma = getPrismaClient() const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) const fractionalRemnants = await prisma.trade.findMany({ where: { exitReason: 'FRACTIONAL_REMNANT' as any, exitTime: { gte: twentyFourHoursAgo } }, select: { id: true, symbol: true, positionSizeUSD: true, exitTime: true } }) if (fractionalRemnants.length > 0) { issues.push(`šŸ›‘ CRITICAL: ${fractionalRemnants.length} fractional position remnant(s) detected!`) issues.push(` These are positions that confirmed close but left small remnants on Drift`) issues.push(` Close attempts exhausted (3 max) - MANUAL INTERVENTION REQUIRED`) for (const remnant of fractionalRemnants) { issues.push(` • ${remnant.symbol}: Size $${remnant.positionSizeUSD.toFixed(2)} (Trade ID: ${remnant.id})`) issues.push(` Detected at: ${remnant.exitTime?.toISOString()}`) issues.push(` Action: Close manually via Drift UI at https://app.drift.trade`) } } const isHealthy = issues.length === 0 return { isHealthy, issues, warnings, info: { dbOpenTrades: dbOpenCount, pmActiveTrades, pmMonitoring, driftPositions, unprotectedPositions } } } catch (error) { issues.push(`āŒ Health check failed: ${error instanceof Error ? error.message : String(error)}`) return { isHealthy: false, issues, warnings, info: { dbOpenTrades: 0, pmActiveTrades: 0, pmMonitoring: false, driftPositions: 0, unprotectedPositions: 0 } } } } /** * Start periodic health checks */ export function startPositionManagerHealthMonitor(): void { console.log('šŸ„ Starting Position Manager health monitor (every 30 seconds)...') // Initial check checkPositionManagerHealth().then(result => { logHealthCheckResult(result) }) // Periodic checks every 30 seconds setInterval(async () => { const result = await checkPositionManagerHealth() // Only log if there are issues or warnings if (!result.isHealthy || result.warnings.length > 0) { logHealthCheckResult(result) } }, 30000) // 30 seconds } function logHealthCheckResult(result: HealthCheckResult): void { if (result.isHealthy && result.warnings.length === 0) { console.log('āœ… Position Manager health check PASSED') console.log(` DB: ${result.info.dbOpenTrades} open | PM: ${result.info.pmActiveTrades} active | Monitoring: ${result.info.pmMonitoring ? 'YES' : 'NO'} | Drift: ${result.info.driftPositions} positions`) return } console.log('\nšŸ„ POSITION MANAGER HEALTH CHECK REPORT:') console.log('━'.repeat(80)) if (result.issues.length > 0) { console.log('\nšŸ”“ CRITICAL ISSUES:') result.issues.forEach(issue => console.log(issue)) } if (result.warnings.length > 0) { console.log('\nāš ļø WARNINGS:') result.warnings.forEach(warning => console.log(warning)) } console.log('\nšŸ“Š SYSTEM STATE:') console.log(` Database open trades: ${result.info.dbOpenTrades}`) console.log(` Position Manager active trades: ${result.info.pmActiveTrades}`) console.log(` Position Manager monitoring: ${result.info.pmMonitoring ? 'āœ… YES' : 'āŒ NO'}`) console.log(` Drift open positions: ${result.info.driftPositions}`) console.log(` Unprotected positions: ${result.info.unprotectedPositions}`) console.log('━'.repeat(80)) }