ba1fe4433e
Changes: - moneyline_v11_2_indicator.pinescript: Alert format now includes SCORE:100 - parse_signal_enhanced.json: Added indicatorScore parsing (SCORE:X regex) - execute/route.ts: Added hasIndicatorScore bypass (score >= 90 bypasses quality check) - Money_Machine.json: Both Execute Trade nodes now pass indicatorScore to API Rationale: v11.2 indicator filters already optimized (2.544 PF, +51.80% return). Bot-side quality scoring was blocking proven profitable signals (e.g., quality 75). Now indicator passes SCORE:100, bot respects it and executes immediately. This completes the signal chain: Indicator (SCORE:100) → n8n parser (indicatorScore) → workflow → bot endpoint (bypass)
413 lines
14 KiB
TypeScript
413 lines
14 KiB
TypeScript
/**
|
||
* Position Manager Health Check
|
||
*
|
||
* CRITICAL: Verifies Position Manager is actually monitoring positions
|
||
*
|
||
* Bug History:
|
||
* - $1,000+ losses because Position Manager logged "added" but never monitored
|
||
* - Silent SL placement failures left positions unprotected
|
||
* - Orphan detection cancelled orders on active positions
|
||
*
|
||
* This health check runs every 30 seconds to detect these critical failures.
|
||
*
|
||
* Created: Dec 8, 2025
|
||
*/
|
||
|
||
import { getInitializedPositionManager } from '../trading/position-manager'
|
||
import { getOpenTrades, getPrismaClient } from '../database/trades'
|
||
import { getDriftService } from '../drift/client'
|
||
import { getMergedConfig } from '../../config/trading'
|
||
|
||
export interface HealthCheckResult {
|
||
isHealthy: boolean
|
||
issues: string[]
|
||
warnings: string[]
|
||
info: {
|
||
dbOpenTrades: number
|
||
pmActiveTrades: number
|
||
pmMonitoring: boolean
|
||
driftPositions: number
|
||
unprotectedPositions: number
|
||
}
|
||
autoSyncTriggered?: boolean
|
||
}
|
||
|
||
// Cooldown to prevent sync spam (5 minutes)
|
||
let lastAutoSyncTime = 0
|
||
const AUTO_SYNC_COOLDOWN_MS = 5 * 60 * 1000
|
||
|
||
/**
|
||
* Automatically trigger position sync when untracked positions detected
|
||
*
|
||
* CRITICAL: This prevents the $1,000 loss scenario where Telegram bot
|
||
* opens positions but Position Manager never tracks them.
|
||
*
|
||
* Cooldown: 5 minutes between auto-syncs to prevent spam
|
||
*/
|
||
async function autoSyncUntrackedPositions(): Promise<boolean> {
|
||
const now = Date.now()
|
||
|
||
// Check cooldown
|
||
if (now - lastAutoSyncTime < AUTO_SYNC_COOLDOWN_MS) {
|
||
const remainingSeconds = Math.ceil((AUTO_SYNC_COOLDOWN_MS - (now - lastAutoSyncTime)) / 1000)
|
||
console.log(`⏳ Auto-sync cooldown active (${remainingSeconds}s remaining)`)
|
||
return false
|
||
}
|
||
|
||
try {
|
||
console.log('🔄 AUTO-SYNC TRIGGERED: Untracked positions detected, syncing with Drift...')
|
||
|
||
// Call the sync endpoint internally
|
||
const pm = await getInitializedPositionManager()
|
||
const driftService = getDriftService()
|
||
const driftPositions = await driftService.getAllPositions()
|
||
|
||
console.log(`📊 Found ${driftPositions.length} positions on Drift`)
|
||
|
||
let added = 0
|
||
for (const driftPos of driftPositions) {
|
||
const isTracked = Array.from((pm as any).activeTrades.values()).some(
|
||
(t: any) => t.symbol === driftPos.symbol
|
||
)
|
||
|
||
if (!isTracked && Math.abs(driftPos.size) > 0) {
|
||
console.log(`➕ Auto-adding ${driftPos.symbol} to Position Manager`)
|
||
|
||
// Import sync logic
|
||
const { syncSinglePosition } = await import('../trading/sync-helper')
|
||
await syncSinglePosition(driftPos, pm)
|
||
added++
|
||
}
|
||
}
|
||
|
||
lastAutoSyncTime = now
|
||
console.log(`✅ AUTO-SYNC COMPLETE: Added ${added} position(s) to monitoring`)
|
||
|
||
return true
|
||
} catch (error) {
|
||
console.error('❌ Auto-sync failed:', error)
|
||
return false
|
||
}
|
||
}
|
||
|
||
function calculatePrice(entry: number, percent: number, direction: 'long' | 'short'): number {
|
||
return direction === 'long'
|
||
? entry * (1 + percent / 100)
|
||
: entry * (1 - percent / 100)
|
||
}
|
||
|
||
async function ensureExitOrdersForTrade(
|
||
trade: any,
|
||
config: ReturnType<typeof getMergedConfig>
|
||
): Promise<{ placed: boolean; message?: string; error?: string }> {
|
||
try {
|
||
const positionSizeUSD = trade.positionSizeUSD || trade.positionSize || 0
|
||
if (!positionSizeUSD || !trade.entryPrice) {
|
||
return { placed: false, error: 'Missing position size or entry price' }
|
||
}
|
||
|
||
const direction: 'long' | 'short' = trade.direction === 'short' ? 'short' : 'long'
|
||
|
||
const tp1Price =
|
||
trade.takeProfit1Price || calculatePrice(trade.entryPrice, config.takeProfit1Percent, direction)
|
||
const tp2Price =
|
||
trade.takeProfit2Price || calculatePrice(trade.entryPrice, config.takeProfit2Percent, direction)
|
||
const stopLossPrice =
|
||
trade.stopLossPrice || calculatePrice(trade.entryPrice, config.stopLossPercent, direction)
|
||
|
||
const tp1SizePercent = trade.tp1SizePercent ?? config.takeProfit1SizePercent
|
||
const tp2SizePercentRaw = trade.tp2SizePercent ?? config.takeProfit2SizePercent ?? 0
|
||
const tp2SizePercent = config.useTp2AsTriggerOnly && tp2SizePercentRaw <= 0 ? 0 : tp2SizePercentRaw
|
||
|
||
const softStopPrice = config.useDualStops
|
||
? calculatePrice(trade.entryPrice, config.softStopPercent, direction)
|
||
: undefined
|
||
const hardStopPrice = config.useDualStops
|
||
? calculatePrice(trade.entryPrice, config.hardStopPercent, direction)
|
||
: undefined
|
||
|
||
const { placeExitOrders } = await import('../drift/orders')
|
||
|
||
const placeResult = await placeExitOrders({
|
||
symbol: trade.symbol,
|
||
positionSizeUSD,
|
||
entryPrice: trade.entryPrice,
|
||
tp1Price,
|
||
tp2Price,
|
||
stopLossPrice,
|
||
tp1SizePercent,
|
||
tp2SizePercent,
|
||
direction,
|
||
useDualStops: config.useDualStops,
|
||
softStopPrice,
|
||
softStopBuffer: config.softStopBuffer,
|
||
hardStopPrice,
|
||
})
|
||
|
||
if (!placeResult.success) {
|
||
return { placed: false, error: placeResult.error || 'Unknown error placing exit orders' }
|
||
}
|
||
|
||
const signatures = placeResult.signatures || []
|
||
const normalizedTp2Percent = tp2SizePercent === undefined ? 100 : Math.max(0, tp2SizePercent)
|
||
const tp1USD = (positionSizeUSD * tp1SizePercent) / 100
|
||
const remainingAfterTP1 = positionSizeUSD - tp1USD
|
||
const tp2USD = (remainingAfterTP1 * normalizedTp2Percent) / 100
|
||
|
||
let idx = 0
|
||
const updateData: any = {}
|
||
|
||
if (tp1USD > 0 && idx < signatures.length) {
|
||
updateData.tp1OrderTx = signatures[idx++]
|
||
}
|
||
if (normalizedTp2Percent > 0 && idx < signatures.length) {
|
||
updateData.tp2OrderTx = signatures[idx++]
|
||
}
|
||
|
||
if (config.useDualStops && softStopPrice && hardStopPrice) {
|
||
if (idx < signatures.length) {
|
||
updateData.softStopOrderTx = signatures[idx++]
|
||
}
|
||
if (idx < signatures.length) {
|
||
updateData.hardStopOrderTx = signatures[idx++]
|
||
}
|
||
} else if (idx < signatures.length) {
|
||
updateData.slOrderTx = signatures[idx++]
|
||
}
|
||
|
||
const prisma = getPrismaClient()
|
||
await prisma.trade.update({ where: { id: trade.id }, data: updateData })
|
||
|
||
return { placed: true, message: 'Protective exits placed' }
|
||
} catch (error) {
|
||
return {
|
||
placed: false,
|
||
error: error instanceof Error ? error.message : String(error)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check Position Manager health
|
||
*
|
||
* CRITICAL CHECKS:
|
||
* 1. If DB has open trades, Position Manager MUST be monitoring
|
||
* 2. If Position Manager has trades, monitoring MUST be active
|
||
* 3. All open positions MUST have TP/SL orders on-chain
|
||
* 4. Position Manager trade count MUST match Drift position count
|
||
*/
|
||
export async function checkPositionManagerHealth(): Promise<HealthCheckResult> {
|
||
const issues: string[] = []
|
||
const warnings: string[] = []
|
||
const config = getMergedConfig()
|
||
|
||
try {
|
||
// Get database open trades
|
||
const dbTrades = await getOpenTrades()
|
||
const dbOpenCount = dbTrades.length
|
||
|
||
// Get Position Manager state
|
||
const pm = await getInitializedPositionManager()
|
||
const pmState = (pm as any)
|
||
let pmActiveTrades = pmState.activeTrades?.size || 0
|
||
let pmMonitoring = pmState.isMonitoring || false
|
||
|
||
// Get Drift positions
|
||
const driftService = getDriftService()
|
||
const positions = await driftService.getAllPositions()
|
||
const driftPositions = positions.filter(p => Math.abs(p.size) > 0).length
|
||
|
||
// CRITICAL CHECK #1: DB has open trades but PM not monitoring
|
||
if (dbOpenCount > 0 && !pmMonitoring) {
|
||
console.log('🛠️ Health monitor: Attempting automatic monitoring restore from DB...')
|
||
try {
|
||
await pm.initialize(true)
|
||
pmActiveTrades = (pm as any).activeTrades?.size || 0
|
||
pmMonitoring = (pm as any).isMonitoring || false
|
||
} catch (restoreError) {
|
||
console.error('❌ Failed to auto-restore monitoring:', restoreError)
|
||
}
|
||
}
|
||
|
||
// Re-check after attempted restore
|
||
if (dbOpenCount > 0 && !pmMonitoring) {
|
||
issues.push(`❌ CRITICAL: ${dbOpenCount} open trades in DB but Position Manager NOT monitoring!`)
|
||
issues.push(` This means NO TP/SL protection, NO monitoring, UNCONTROLLED RISK`)
|
||
issues.push(` ACTION REQUIRED: Restart container to restore monitoring`)
|
||
}
|
||
|
||
// CRITICAL CHECK #2: PM has trades but not monitoring
|
||
if (pmActiveTrades > 0 && !pmMonitoring) {
|
||
issues.push(`❌ CRITICAL: Position Manager has ${pmActiveTrades} active trades but monitoring is OFF!`)
|
||
issues.push(` This is the $1,000 loss bug - trades "added" but never monitored`)
|
||
issues.push(` ACTION REQUIRED: Fix startMonitoring() function`)
|
||
}
|
||
|
||
// CRITICAL CHECK #3: DB vs PM mismatch
|
||
if (dbOpenCount !== pmActiveTrades) {
|
||
warnings.push(`⚠️ WARNING: DB has ${dbOpenCount} open trades, PM has ${pmActiveTrades} active trades`)
|
||
warnings.push(` Possible orphaned position or monitoring not started`)
|
||
}
|
||
|
||
// CRITICAL CHECK #4: PM vs Drift mismatch
|
||
if (pmActiveTrades !== driftPositions) {
|
||
warnings.push(`⚠️ WARNING: Position Manager has ${pmActiveTrades} trades, Drift has ${driftPositions} positions`)
|
||
warnings.push(` Possible untracked position or external closure`)
|
||
|
||
// AUTO-SYNC: If Drift has MORE positions than PM, sync automatically
|
||
if (driftPositions > pmActiveTrades) {
|
||
console.log(`🚨 UNTRACKED POSITIONS DETECTED: Drift has ${driftPositions}, PM has ${pmActiveTrades}`)
|
||
const synced = await autoSyncUntrackedPositions()
|
||
|
||
if (synced) {
|
||
warnings.push(`✅ AUTO-SYNC: Triggered position sync to restore protection`)
|
||
// Re-check PM state after sync
|
||
pmActiveTrades = (pm as any).activeTrades?.size || 0
|
||
}
|
||
}
|
||
}
|
||
|
||
// Check for unprotected positions
|
||
let unprotectedPositions = 0
|
||
for (const trade of dbTrades) {
|
||
const hasDbSignatures = !!(trade.slOrderTx || trade.softStopOrderTx || trade.hardStopOrderTx)
|
||
|
||
if (!hasDbSignatures) {
|
||
unprotectedPositions++
|
||
issues.push(`❌ CRITICAL: Position ${trade.symbol} (${trade.id}) has NO STOP LOSS ORDERS!`)
|
||
issues.push(` Entry: $${trade.entryPrice}, Size: $${trade.positionSizeUSD}`)
|
||
issues.push(` Attempting automatic protective order placement...`)
|
||
|
||
const remediation = await ensureExitOrdersForTrade(trade, config)
|
||
if (remediation.placed) {
|
||
warnings.push(`✅ Auto-placed protective exit orders for ${trade.symbol} (${trade.id})`)
|
||
} else if (remediation.error) {
|
||
issues.push(` ❌ Failed to auto-place exits: ${remediation.error}`)
|
||
}
|
||
}
|
||
|
||
if (!trade.tp1OrderTx) {
|
||
warnings.push(`⚠️ Position ${trade.symbol} missing TP1 order (not synced)`)
|
||
}
|
||
|
||
if (!trade.tp2OrderTx) {
|
||
warnings.push(`⚠️ Position ${trade.symbol} missing TP2 order (not synced)`)
|
||
}
|
||
}
|
||
|
||
// BUG #89 FIX PART 3 (Dec 16, 2025): Check for fractional position remnants
|
||
// Query database for positions marked as FRACTIONAL_REMNANT in last 24 hours
|
||
// These require manual intervention via Drift UI
|
||
const prisma = getPrismaClient()
|
||
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
|
||
|
||
const fractionalRemnants = await prisma.trade.findMany({
|
||
where: {
|
||
exitReason: 'FRACTIONAL_REMNANT' as any,
|
||
exitTime: { gte: twentyFourHoursAgo }
|
||
},
|
||
select: {
|
||
id: true,
|
||
symbol: true,
|
||
positionSizeUSD: true,
|
||
exitTime: true
|
||
}
|
||
})
|
||
|
||
if (fractionalRemnants.length > 0) {
|
||
issues.push(`🛑 CRITICAL: ${fractionalRemnants.length} fractional position remnant(s) detected!`)
|
||
issues.push(` These are positions that confirmed close but left small remnants on Drift`)
|
||
issues.push(` Close attempts exhausted (3 max) - MANUAL INTERVENTION REQUIRED`)
|
||
|
||
for (const remnant of fractionalRemnants) {
|
||
issues.push(` • ${remnant.symbol}: Size $${remnant.positionSizeUSD.toFixed(2)} (Trade ID: ${remnant.id})`)
|
||
issues.push(` Detected at: ${remnant.exitTime?.toISOString()}`)
|
||
issues.push(` Action: Close manually via Drift UI at https://app.drift.trade`)
|
||
}
|
||
}
|
||
|
||
const isHealthy = issues.length === 0
|
||
|
||
return {
|
||
isHealthy,
|
||
issues,
|
||
warnings,
|
||
info: {
|
||
dbOpenTrades: dbOpenCount,
|
||
pmActiveTrades,
|
||
pmMonitoring,
|
||
driftPositions,
|
||
unprotectedPositions
|
||
}
|
||
}
|
||
|
||
} catch (error) {
|
||
issues.push(`❌ Health check failed: ${error instanceof Error ? error.message : String(error)}`)
|
||
|
||
return {
|
||
isHealthy: false,
|
||
issues,
|
||
warnings,
|
||
info: {
|
||
dbOpenTrades: 0,
|
||
pmActiveTrades: 0,
|
||
pmMonitoring: false,
|
||
driftPositions: 0,
|
||
unprotectedPositions: 0
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Start periodic health checks
|
||
*/
|
||
export function startPositionManagerHealthMonitor(): void {
|
||
console.log('🏥 Starting Position Manager health monitor (every 30 seconds)...')
|
||
|
||
// Initial check
|
||
checkPositionManagerHealth().then(result => {
|
||
logHealthCheckResult(result)
|
||
})
|
||
|
||
// Periodic checks every 30 seconds
|
||
setInterval(async () => {
|
||
const result = await checkPositionManagerHealth()
|
||
|
||
// Only log if there are issues or warnings
|
||
if (!result.isHealthy || result.warnings.length > 0) {
|
||
logHealthCheckResult(result)
|
||
}
|
||
}, 30000) // 30 seconds
|
||
}
|
||
|
||
function logHealthCheckResult(result: HealthCheckResult): void {
|
||
if (result.isHealthy && result.warnings.length === 0) {
|
||
console.log('✅ Position Manager health check PASSED')
|
||
console.log(` DB: ${result.info.dbOpenTrades} open | PM: ${result.info.pmActiveTrades} active | Monitoring: ${result.info.pmMonitoring ? 'YES' : 'NO'} | Drift: ${result.info.driftPositions} positions`)
|
||
return
|
||
}
|
||
|
||
console.log('\n🏥 POSITION MANAGER HEALTH CHECK REPORT:')
|
||
console.log('━'.repeat(80))
|
||
|
||
if (result.issues.length > 0) {
|
||
console.log('\n🔴 CRITICAL ISSUES:')
|
||
result.issues.forEach(issue => console.log(issue))
|
||
}
|
||
|
||
if (result.warnings.length > 0) {
|
||
console.log('\n⚠️ WARNINGS:')
|
||
result.warnings.forEach(warning => console.log(warning))
|
||
}
|
||
|
||
console.log('\n📊 SYSTEM STATE:')
|
||
console.log(` Database open trades: ${result.info.dbOpenTrades}`)
|
||
console.log(` Position Manager active trades: ${result.info.pmActiveTrades}`)
|
||
console.log(` Position Manager monitoring: ${result.info.pmMonitoring ? '✅ YES' : '❌ NO'}`)
|
||
console.log(` Drift open positions: ${result.info.driftPositions}`)
|
||
console.log(` Unprotected positions: ${result.info.unprotectedPositions}`)
|
||
|
||
console.log('━'.repeat(80))
|
||
}
|