/** * Position Manager Health Check * * CRITICAL: Verifies Position Manager is actually monitoring positions * * Bug History: * - $1,000+ losses because Position Manager logged "added" but never monitored * - Silent SL placement failures left positions unprotected * - Orphan detection cancelled orders on active positions * * This health check runs every 30 seconds to detect these critical failures. * * Created: Dec 8, 2025 */ import { getInitializedPositionManager } from '../trading/position-manager' import { getOpenTrades } from '../database/trades' import { getDriftService } from '../drift/client' export interface HealthCheckResult { isHealthy: boolean issues: string[] warnings: string[] info: { dbOpenTrades: number pmActiveTrades: number pmMonitoring: boolean driftPositions: number unprotectedPositions: number } autoSyncTriggered?: boolean } // Cooldown to prevent sync spam (5 minutes) let lastAutoSyncTime = 0 const AUTO_SYNC_COOLDOWN_MS = 5 * 60 * 1000 /** * Automatically trigger position sync when untracked positions detected * * CRITICAL: This prevents the $1,000 loss scenario where Telegram bot * opens positions but Position Manager never tracks them. * * Cooldown: 5 minutes between auto-syncs to prevent spam */ async function autoSyncUntrackedPositions(): Promise { const now = Date.now() // Check cooldown if (now - lastAutoSyncTime < AUTO_SYNC_COOLDOWN_MS) { const remainingSeconds = Math.ceil((AUTO_SYNC_COOLDOWN_MS - (now - lastAutoSyncTime)) / 1000) console.log(`ā³ Auto-sync cooldown active (${remainingSeconds}s remaining)`) return false } try { console.log('šŸ”„ AUTO-SYNC TRIGGERED: Untracked positions detected, syncing with Drift...') // Call the sync endpoint internally const pm = await getInitializedPositionManager() const driftService = getDriftService() const driftPositions = await driftService.getAllPositions() console.log(`šŸ“Š Found ${driftPositions.length} positions on Drift`) let added = 0 for (const driftPos of driftPositions) { const isTracked = Array.from((pm as any).activeTrades.values()).some( (t: any) => t.symbol === driftPos.symbol ) if (!isTracked && Math.abs(driftPos.size) > 0) { console.log(`āž• Auto-adding ${driftPos.symbol} to Position Manager`) // Import sync logic const { syncSinglePosition } = await import('../trading/sync-helper') await syncSinglePosition(driftPos, pm) added++ } } lastAutoSyncTime = now console.log(`āœ… AUTO-SYNC COMPLETE: Added ${added} position(s) to monitoring`) return true } catch (error) { console.error('āŒ Auto-sync failed:', error) return false } } /** * Check Position Manager health * * CRITICAL CHECKS: * 1. If DB has open trades, Position Manager MUST be monitoring * 2. If Position Manager has trades, monitoring MUST be active * 3. All open positions MUST have TP/SL orders on-chain * 4. Position Manager trade count MUST match Drift position count */ export async function checkPositionManagerHealth(): Promise { const issues: string[] = [] const warnings: string[] = [] try { // Get database open trades const dbTrades = await getOpenTrades() const dbOpenCount = dbTrades.length // Get Position Manager state const pm = await getInitializedPositionManager() const pmState = (pm as any) let pmActiveTrades = pmState.activeTrades?.size || 0 let pmMonitoring = pmState.isMonitoring || false // Get Drift positions const driftService = getDriftService() const positions = await driftService.getAllPositions() const driftPositions = positions.filter(p => Math.abs(p.size) > 0).length // CRITICAL CHECK #1: DB has open trades but PM not monitoring if (dbOpenCount > 0 && !pmMonitoring) { console.log('šŸ› ļø Health monitor: Attempting automatic monitoring restore from DB...') try { await pm.initialize(true) pmActiveTrades = (pm as any).activeTrades?.size || 0 pmMonitoring = (pm as any).isMonitoring || false } catch (restoreError) { console.error('āŒ Failed to auto-restore monitoring:', restoreError) } } // Re-check after attempted restore if (dbOpenCount > 0 && !pmMonitoring) { issues.push(`āŒ CRITICAL: ${dbOpenCount} open trades in DB but Position Manager NOT monitoring!`) issues.push(` This means NO TP/SL protection, NO monitoring, UNCONTROLLED RISK`) issues.push(` ACTION REQUIRED: Restart container to restore monitoring`) } // CRITICAL CHECK #2: PM has trades but not monitoring if (pmActiveTrades > 0 && !pmMonitoring) { issues.push(`āŒ CRITICAL: Position Manager has ${pmActiveTrades} active trades but monitoring is OFF!`) issues.push(` This is the $1,000 loss bug - trades "added" but never monitored`) issues.push(` ACTION REQUIRED: Fix startMonitoring() function`) } // CRITICAL CHECK #3: DB vs PM mismatch if (dbOpenCount !== pmActiveTrades) { warnings.push(`āš ļø WARNING: DB has ${dbOpenCount} open trades, PM has ${pmActiveTrades} active trades`) warnings.push(` Possible orphaned position or monitoring not started`) } // CRITICAL CHECK #4: PM vs Drift mismatch if (pmActiveTrades !== driftPositions) { warnings.push(`āš ļø WARNING: Position Manager has ${pmActiveTrades} trades, Drift has ${driftPositions} positions`) warnings.push(` Possible untracked position or external closure`) // AUTO-SYNC: If Drift has MORE positions than PM, sync automatically if (driftPositions > pmActiveTrades) { console.log(`🚨 UNTRACKED POSITIONS DETECTED: Drift has ${driftPositions}, PM has ${pmActiveTrades}`) const synced = await autoSyncUntrackedPositions() if (synced) { warnings.push(`āœ… AUTO-SYNC: Triggered position sync to restore protection`) // Re-check PM state after sync pmActiveTrades = (pm as any).activeTrades?.size || 0 } } } // Check for unprotected positions // NOTE: Synced/placeholder positions (signalSource='autosync') have NULL signatures in DB // but orders exist on Drift. Position Manager monitoring provides backup protection. let unprotectedPositions = 0 for (const trade of dbTrades) { const hasDbSignatures = !!(trade.slOrderTx || trade.softStopOrderTx || trade.hardStopOrderTx) const isSyncedPosition = trade.signalSource === 'autosync' || trade.timeframe === 'sync' if (!hasDbSignatures && !isSyncedPosition) { // This is NOT a synced position but has no SL orders - CRITICAL unprotectedPositions++ issues.push(`āŒ CRITICAL: Position ${trade.symbol} (${trade.id}) has NO STOP LOSS ORDERS!`) issues.push(` Entry: $${trade.entryPrice}, Size: $${trade.positionSizeUSD}`) issues.push(` This is the silent SL placement failure bug`) } if (!trade.tp1OrderTx && !isSyncedPosition) { warnings.push(`āš ļø Position ${trade.symbol} missing TP1 order (not synced)`) } if (!trade.tp2OrderTx && !isSyncedPosition) { warnings.push(`āš ļø Position ${trade.symbol} missing TP2 order (not synced)`) } } const isHealthy = issues.length === 0 return { isHealthy, issues, warnings, info: { dbOpenTrades: dbOpenCount, pmActiveTrades, pmMonitoring, driftPositions, unprotectedPositions } } } catch (error) { issues.push(`āŒ Health check failed: ${error instanceof Error ? error.message : String(error)}`) return { isHealthy: false, issues, warnings, info: { dbOpenTrades: 0, pmActiveTrades: 0, pmMonitoring: false, driftPositions: 0, unprotectedPositions: 0 } } } } /** * Start periodic health checks */ export function startPositionManagerHealthMonitor(): void { console.log('šŸ„ Starting Position Manager health monitor (every 30 seconds)...') // Initial check checkPositionManagerHealth().then(result => { logHealthCheckResult(result) }) // Periodic checks every 30 seconds setInterval(async () => { const result = await checkPositionManagerHealth() // Only log if there are issues or warnings if (!result.isHealthy || result.warnings.length > 0) { logHealthCheckResult(result) } }, 30000) // 30 seconds } function logHealthCheckResult(result: HealthCheckResult): void { if (result.isHealthy && result.warnings.length === 0) { console.log('āœ… Position Manager health check PASSED') console.log(` DB: ${result.info.dbOpenTrades} open | PM: ${result.info.pmActiveTrades} active | Monitoring: ${result.info.pmMonitoring ? 'YES' : 'NO'} | Drift: ${result.info.driftPositions} positions`) return } console.log('\nšŸ„ POSITION MANAGER HEALTH CHECK REPORT:') console.log('━'.repeat(80)) if (result.issues.length > 0) { console.log('\nšŸ”“ CRITICAL ISSUES:') result.issues.forEach(issue => console.log(issue)) } if (result.warnings.length > 0) { console.log('\nāš ļø WARNINGS:') result.warnings.forEach(warning => console.log(warning)) } console.log('\nšŸ“Š SYSTEM STATE:') console.log(` Database open trades: ${result.info.dbOpenTrades}`) console.log(` Position Manager active trades: ${result.info.pmActiveTrades}`) console.log(` Position Manager monitoring: ${result.info.pmMonitoring ? 'āœ… YES' : 'āŒ NO'}`) console.log(` Drift open positions: ${result.info.driftPositions}`) console.log(` Unprotected positions: ${result.info.unprotectedPositions}`) console.log('━'.repeat(80)) }