f6c9a7b7a4
- logger.log is silenced in production (NODE_ENV=production) - Service initialization logs were hidden even though services were starting - Changed to console.log for visibility in production logs - Affects: data cleanup, blocked signal tracker, stop hunt tracker, smart validation
466 lines
18 KiB
TypeScript
466 lines
18 KiB
TypeScript
/**
|
|
* Position Manager Startup Initialization
|
|
*
|
|
* Ensures Position Manager starts monitoring on bot startup
|
|
* This prevents orphaned trades when the bot restarts
|
|
*/
|
|
|
|
import { getInitializedPositionManager } from '../trading/position-manager'
|
|
import { logger } from '../utils/logger'
|
|
import { initializeDriftService } from '../drift/client'
|
|
import { getPrismaClient, createTrade } from '../database/trades'
|
|
import { getMarketConfig, getMergedConfig } from '../../config/trading'
|
|
import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker'
|
|
import { startStopHuntTracking } from '../trading/stop-hunt-tracker'
|
|
import { startSmartValidation } from '../trading/smart-validation-queue'
|
|
import { startDataCleanup } from '../maintenance/data-cleanup'
|
|
import { logCriticalError } from '../utils/persistent-logger'
|
|
import { sendPositionClosedNotification } from '../notifications/telegram'
|
|
|
|
let initStarted = false
|
|
|
|
export async function initializePositionManagerOnStartup() {
|
|
if (initStarted) {
|
|
return
|
|
}
|
|
|
|
initStarted = true
|
|
|
|
logger.log('🚀 Initializing Position Manager on startup...')
|
|
|
|
try {
|
|
// CRITICAL (Dec 5, 2025): Start services FIRST, before validation
|
|
// Bug fix: validateOpenTrades() returns early if no trades, skipping services
|
|
// Services must run regardless of open trades count
|
|
|
|
// CRITICAL (Dec 2, 2025): Start data cleanup service for 4-week retention
|
|
// User directive: "we want to store the data for 4 weeks"
|
|
// Runs daily at 3 AM to delete MarketData records older than 28 days
|
|
console.log('🧹 Starting data cleanup service...')
|
|
startDataCleanup()
|
|
|
|
// Start blocked signal price tracking
|
|
console.log('🔬 Starting blocked signal price tracker...')
|
|
startBlockedSignalTracking()
|
|
|
|
// Start stop hunt revenge tracker
|
|
console.log('🎯 Starting stop hunt revenge tracker...')
|
|
await startStopHuntTracking()
|
|
|
|
// Start smart entry validation queue (Nov 30, 2025)
|
|
console.log('🧠 Starting smart entry validation system...')
|
|
await startSmartValidation()
|
|
|
|
// CRITICAL: Run database sync validator to clean up duplicates
|
|
const { validateAllOpenTrades } = await import('../database/sync-validator')
|
|
console.log('🔍 Running database sync validation before Position Manager init...')
|
|
const validationResult = await validateAllOpenTrades()
|
|
|
|
if (validationResult.ghosts > 0) {
|
|
logger.log(`✅ Cleaned up ${validationResult.ghosts} ghost/duplicate trades`)
|
|
}
|
|
|
|
// Then validate open trades against Drift positions
|
|
await validateOpenTrades()
|
|
|
|
// CRITICAL: Detect orphaned positions (on Drift but not in database)
|
|
await detectOrphanedPositions()
|
|
|
|
const manager = await getInitializedPositionManager()
|
|
const status = manager.getStatus()
|
|
|
|
console.log(`✅ Position Manager ready - ${status.activeTradesCount} active trades`)
|
|
|
|
if (status.activeTradesCount > 0) {
|
|
console.log(`📊 Monitoring: ${status.symbols.join(', ')}`)
|
|
}
|
|
} catch (error) {
|
|
console.error('❌ Failed to initialize Position Manager on startup:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validate that open trades in database match actual Drift positions
|
|
*
|
|
* CRITICAL FIX (Nov 14, 2025):
|
|
* - Also checks trades marked as "closed" in DB that might still be open on Drift
|
|
* - Happens when close transaction fails but bot marks it as closed anyway
|
|
* - Restores Position Manager tracking for these orphaned positions
|
|
*/
|
|
async function validateOpenTrades() {
|
|
try {
|
|
const prisma = getPrismaClient()
|
|
|
|
// Get both truly open trades AND recently "closed" trades (last 24h)
|
|
// Recently closed trades might still be open if close transaction failed
|
|
// TEMPORARILY REDUCED: Check only last 5 closed trades to avoid rate limiting on startup
|
|
const [openTrades, recentlyClosedTrades] = await Promise.all([
|
|
prisma.trade.findMany({
|
|
where: { status: 'open' },
|
|
orderBy: { entryTime: 'asc' }
|
|
}),
|
|
prisma.trade.findMany({
|
|
where: {
|
|
exitReason: { not: null },
|
|
exitTime: { gte: new Date(Date.now() - 6 * 60 * 60 * 1000) } // Last 6 hours (reduced from 24h)
|
|
},
|
|
orderBy: { exitTime: 'desc' },
|
|
take: 5 // Reduced from 20 to avoid rate limiting
|
|
})
|
|
])
|
|
|
|
const allTradesToCheck = [...openTrades, ...recentlyClosedTrades]
|
|
|
|
if (allTradesToCheck.length === 0) {
|
|
logger.log('✅ No open trades to validate')
|
|
return
|
|
}
|
|
|
|
logger.log(`🔍 Validating ${openTrades.length} open + ${recentlyClosedTrades.length} recently closed trades against Drift...`)
|
|
|
|
// CRITICAL: Group trades by symbol to handle multiple DB entries for same Drift position
|
|
// This prevents reopening old closed trades when only the most recent should be restored
|
|
const tradesBySymbol = new Map<string, any[]>()
|
|
for (const trade of allTradesToCheck) {
|
|
const existing = tradesBySymbol.get(trade.symbol) || []
|
|
existing.push(trade)
|
|
tradesBySymbol.set(trade.symbol, existing)
|
|
}
|
|
|
|
const driftService = await initializeDriftService()
|
|
const driftPositions = await driftService.getAllPositions() // Get all positions once
|
|
|
|
// Process each symbol's trades (keep only most recent if multiple exist)
|
|
for (const [symbol, trades] of tradesBySymbol) {
|
|
// Sort by creation time, newest first
|
|
trades.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
|
|
const mostRecentTrade = trades[0]
|
|
const olderTrades = trades.slice(1)
|
|
|
|
// Close any older trades BEFORE validating the most recent
|
|
for (const oldTrade of olderTrades) {
|
|
if (oldTrade.exitReason === null) {
|
|
logger.log(`🗑️ Closing duplicate old trade: ${oldTrade.id} (${symbol}, created ${oldTrade.createdAt.toISOString()})`)
|
|
await prisma.trade.update({
|
|
where: { id: oldTrade.id },
|
|
data: {
|
|
status: 'closed',
|
|
exitTime: new Date(),
|
|
exitReason: 'DUPLICATE_CLEANUP',
|
|
exitPrice: oldTrade.entryPrice,
|
|
realizedPnL: 0,
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Now validate only the most recent trade for this symbol
|
|
const trade = mostRecentTrade
|
|
|
|
try {
|
|
const marketConfig = getMarketConfig(trade.symbol)
|
|
|
|
// Find matching Drift position by symbol
|
|
const position = driftPositions.find(p => p.symbol === trade.symbol)
|
|
|
|
if (!position || position.size === 0) {
|
|
// No position on Drift
|
|
if (trade.status === 'open') {
|
|
logger.log(`⚠️ PHANTOM TRADE: ${trade.symbol} marked open in DB but not found on Drift`)
|
|
logger.log(` 🗑️ Auto-closing phantom trade...`)
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'closed',
|
|
exitTime: new Date(),
|
|
exitReason: 'PHANTOM_TRADE_CLEANUP',
|
|
exitPrice: trade.entryPrice,
|
|
realizedPnL: 0,
|
|
realizedPnLPercent: 0,
|
|
}
|
|
})
|
|
}
|
|
// If already closed in DB and not on Drift, that's correct - skip
|
|
continue
|
|
}
|
|
|
|
// Position EXISTS on Drift
|
|
const driftDirection = position.side.toLowerCase() as 'long' | 'short'
|
|
|
|
if (driftDirection !== trade.direction) {
|
|
logger.log(`⚠️ DIRECTION MISMATCH: ${trade.symbol} DB=${trade.direction} Drift=${driftDirection}`)
|
|
continue
|
|
}
|
|
|
|
// CRITICAL: If DB says closed but Drift shows open, restore tracking!
|
|
if (trade.exitReason !== null) {
|
|
logger.log(`🔴 CRITICAL: ${trade.symbol} marked as CLOSED in DB but still OPEN on Drift!`)
|
|
logger.log(` DB entry: $${trade.entryPrice.toFixed(2)} | Drift entry: $${position.entryPrice.toFixed(2)}`)
|
|
logger.log(` DB exit: ${trade.exitReason} at ${trade.exitTime?.toISOString()}`)
|
|
logger.log(` Drift: ${position.size} ${trade.symbol} ${driftDirection} @ $${position.entryPrice.toFixed(2)}`)
|
|
logger.log(` 🔄 Reopening trade and correcting entry price to match Drift...`)
|
|
|
|
// Calculate position size in USD using Drift's entry price
|
|
const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex)
|
|
const positionSizeUSD = position.size * currentPrice
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'open',
|
|
exitReason: null,
|
|
exitTime: null,
|
|
exitPrice: null,
|
|
entryPrice: position.entryPrice, // CRITICAL: Use Drift's actual entry price, not DB value
|
|
positionSizeUSD: positionSizeUSD, // Update to current size (may be runner after TP1)
|
|
// Keep original realizedPnL from partial closes if any
|
|
}
|
|
})
|
|
|
|
logger.log(` ✅ Trade restored with corrected entry: $${position.entryPrice.toFixed(2)} (was $${trade.entryPrice.toFixed(2)})`)
|
|
} else {
|
|
logger.log(`✅ ${trade.symbol} ${trade.direction}: Position verified on Drift`)
|
|
}
|
|
|
|
// CRITICAL FIX (Nov 16, 2025): Restore missing on-chain orders
|
|
// Ghost position closed at 22:03 because orders were missing after validator cleanup
|
|
// This ensures EVERY verified position has on-chain TP/SL protection
|
|
if (position && Math.abs(position.size) >= 0.01) {
|
|
await restoreOrdersIfMissing(trade, position, driftService, prisma)
|
|
}
|
|
|
|
} catch (posError) {
|
|
console.error(`❌ Error validating trade ${trade.symbol}:`, posError)
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('❌ Error in validateOpenTrades:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore on-chain exit orders if missing (Nov 16, 2025)
|
|
*
|
|
* CRITICAL: After validator cleanups or container restarts, positions may exist
|
|
* on Drift without any on-chain TP/SL orders. This leaves only Position Manager
|
|
* software protection - if bot crashes, position is completely unprotected.
|
|
*
|
|
* This function checks if orders exist and places them if missing.
|
|
*/
|
|
async function restoreOrdersIfMissing(
|
|
trade: any,
|
|
position: any,
|
|
driftService: any,
|
|
prisma: any
|
|
): Promise<void> {
|
|
try {
|
|
// Check if position has any reduce-only orders
|
|
const hasOrders = position.orders && position.orders.length > 0
|
|
|
|
if (hasOrders) {
|
|
logger.log(`✅ ${trade.symbol} has ${position.orders.length} on-chain orders - protection active`)
|
|
return // Orders exist, nothing to do
|
|
}
|
|
|
|
logger.log(`⚠️ ${trade.symbol} has NO on-chain orders - restoring TP/SL protection...`)
|
|
|
|
// Import order placement function
|
|
const { placeExitOrders } = await import('../drift/orders')
|
|
|
|
// Place exit orders using trade's TP/SL prices
|
|
const result = await placeExitOrders({
|
|
symbol: trade.symbol,
|
|
direction: trade.direction,
|
|
entryPrice: trade.entryPrice,
|
|
tp1Price: trade.takeProfit1Price,
|
|
tp2Price: trade.takeProfit2Price,
|
|
stopLossPrice: trade.stopLossPrice,
|
|
positionSizeUSD: trade.positionSizeUSD,
|
|
tp1SizePercent: 75,
|
|
tp2SizePercent: 0, // TP2-as-runner
|
|
})
|
|
|
|
if (result.success) {
|
|
logger.log(`✅ Orders restored for ${trade.symbol}:`)
|
|
logger.log(` TP1: $${trade.takeProfit1Price.toFixed(4)} (75%)`)
|
|
logger.log(` TP2: $${trade.takeProfit2Price.toFixed(4)} (runner trigger)`)
|
|
logger.log(` SL: $${trade.stopLossPrice.toFixed(4)}`)
|
|
logger.log(` TX: ${result.signatures?.[0]?.slice(0, 8)}...`)
|
|
|
|
// Update database with order transaction signatures
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
tp1OrderTx: result.signatures?.[0],
|
|
tp2OrderTx: result.signatures?.[1],
|
|
slOrderTx: result.signatures?.[2],
|
|
}
|
|
})
|
|
} else {
|
|
console.error(`❌ Failed to restore orders for ${trade.symbol}:`, result.error)
|
|
console.error(` 🚨 CRITICAL: Position is unprotected - only Position Manager monitoring active`)
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error(`❌ Error restoring orders for ${trade.symbol}:`, error)
|
|
console.error(` 🚨 CRITICAL: Position may be unprotected`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect orphaned positions - positions on Drift with NO database record
|
|
*
|
|
* CRITICAL FIX (Nov 21, 2025): Prevents ghost positions from database save failures
|
|
*
|
|
* This can happen when:
|
|
* - Database save fails silently during trade execution
|
|
* - Prisma transaction rolls back but no error thrown
|
|
* - Container restart interrupts database save
|
|
*
|
|
* Recovery:
|
|
* 1. Create retroactive database record with current Drift data
|
|
* 2. Send Telegram alert about orphaned position found
|
|
* 3. Add to Position Manager for normal TP/SL monitoring
|
|
*/
|
|
async function detectOrphanedPositions() {
|
|
try {
|
|
const prisma = getPrismaClient()
|
|
const driftService = await initializeDriftService()
|
|
|
|
logger.log('🔍 Checking for orphaned positions on Drift...')
|
|
|
|
// Get all open positions from Drift
|
|
const driftPositions = await driftService.getAllPositions()
|
|
|
|
if (driftPositions.length === 0) {
|
|
logger.log('✅ No positions on Drift')
|
|
return
|
|
}
|
|
|
|
logger.log(`🔍 Found ${driftPositions.length} positions on Drift, checking database...`)
|
|
|
|
// Get all open trades from database
|
|
const openTrades = await prisma.trade.findMany({
|
|
where: { status: 'open' },
|
|
select: { symbol: true, positionId: true, direction: true }
|
|
})
|
|
|
|
const trackedSymbols = new Set(openTrades.map(t => `${t.symbol}-${t.direction}`))
|
|
|
|
// Check each Drift position
|
|
for (const position of driftPositions) {
|
|
const positionKey = `${position.symbol}-${position.side.toLowerCase()}`
|
|
|
|
if (trackedSymbols.has(positionKey)) {
|
|
logger.log(`✅ ${position.symbol} ${position.side} tracked in database`)
|
|
continue
|
|
}
|
|
|
|
// ORPHAN DETECTED!
|
|
// Get current price from Drift oracle
|
|
const marketConfig = getMarketConfig(position.symbol)
|
|
const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex)
|
|
const positionSizeUSD = Math.abs(position.size) * currentPrice
|
|
|
|
logger.log(`🚨 ORPHAN POSITION DETECTED!`)
|
|
logger.log(` Symbol: ${position.symbol}`)
|
|
logger.log(` Direction: ${position.side}`)
|
|
logger.log(` Size: ${Math.abs(position.size)} (notional: $${positionSizeUSD.toFixed(2)})`)
|
|
logger.log(` Entry: $${position.entryPrice.toFixed(4)}`)
|
|
logger.log(` Current: $${currentPrice.toFixed(4)}`)
|
|
|
|
// Log to persistent file
|
|
logCriticalError('ORPHAN POSITION DETECTED - Creating retroactive database record', {
|
|
symbol: position.symbol,
|
|
direction: position.side.toLowerCase(),
|
|
entryPrice: position.entryPrice,
|
|
size: Math.abs(position.size),
|
|
currentPrice: currentPrice,
|
|
detectedAt: new Date().toISOString()
|
|
})
|
|
|
|
try {
|
|
// Get config for TP/SL calculation
|
|
const config = getMergedConfig()
|
|
|
|
// Calculate estimated TP/SL prices based on current config
|
|
const direction = position.side.toLowerCase() as 'long' | 'short'
|
|
const entryPrice = position.entryPrice
|
|
|
|
// Calculate TP/SL using same logic as execute endpoint
|
|
const stopLossPrice = direction === 'long'
|
|
? entryPrice * (1 + config.stopLossPercent / 100)
|
|
: entryPrice * (1 - config.stopLossPercent / 100)
|
|
|
|
const tp1Price = direction === 'long'
|
|
? entryPrice * (1 + config.takeProfit1Percent / 100)
|
|
: entryPrice * (1 - config.takeProfit1Percent / 100)
|
|
|
|
const tp2Price = direction === 'long'
|
|
? entryPrice * (1 + config.takeProfit2Percent / 100)
|
|
: entryPrice * (1 - config.takeProfit2Percent / 100)
|
|
|
|
// Create retroactive database record
|
|
logger.log(`🔄 Creating retroactive database record...`)
|
|
const trade = await createTrade({
|
|
positionId: `ORPHAN-${Date.now()}`, // Fake position ID since we don't have transaction
|
|
symbol: position.symbol,
|
|
direction: direction,
|
|
entryPrice: entryPrice,
|
|
positionSizeUSD: positionSizeUSD,
|
|
leverage: config.leverage,
|
|
stopLossPrice: stopLossPrice,
|
|
takeProfit1Price: tp1Price,
|
|
takeProfit2Price: tp2Price,
|
|
tp1SizePercent: config.takeProfit1SizePercent || 75,
|
|
tp2SizePercent: config.takeProfit2SizePercent || 0,
|
|
entryOrderTx: `ORPHAN-${Date.now()}`, // Fake transaction ID
|
|
configSnapshot: config,
|
|
signalSource: 'orphan_recovery',
|
|
timeframe: 'unknown',
|
|
status: 'open',
|
|
})
|
|
|
|
logger.log(`✅ Retroactive database record created: ${trade.id}`)
|
|
|
|
// Send Telegram notification
|
|
try {
|
|
await sendPositionClosedNotification({
|
|
symbol: position.symbol,
|
|
direction: direction,
|
|
entryPrice: entryPrice,
|
|
exitPrice: currentPrice,
|
|
positionSize: positionSizeUSD,
|
|
realizedPnL: 0, // Unknown
|
|
holdTimeSeconds: 0,
|
|
exitReason: 'ORPHAN_DETECTED',
|
|
maxGain: 0,
|
|
maxDrawdown: 0,
|
|
})
|
|
} catch (telegramError) {
|
|
console.error('Failed to send orphan notification:', telegramError)
|
|
}
|
|
|
|
logger.log(`🎯 Orphan position now tracked and monitored`)
|
|
|
|
} catch (recoveryError) {
|
|
console.error(`❌ Failed to recover orphan position ${position.symbol}:`, recoveryError)
|
|
logCriticalError('ORPHAN RECOVERY FAILED', {
|
|
symbol: position.symbol,
|
|
error: recoveryError instanceof Error ? recoveryError.message : String(recoveryError)
|
|
})
|
|
}
|
|
}
|
|
|
|
logger.log('✅ Orphan position detection complete')
|
|
|
|
} catch (error) {
|
|
console.error('❌ Error detecting orphaned positions:', error)
|
|
logCriticalError('Orphan detection failed', {
|
|
error: error instanceof Error ? error.message : String(error)
|
|
})
|
|
}
|
|
}
|