4ab7bf58da
CRITICAL: Position Manager stops monitoring randomly User had to manually close SOL-PERP position after PM stopped at 23:21. Implemented double-checking system to detect when positions marked closed in DB are still open on Drift (and vice versa): 1. DriftStateVerifier service (lib/monitoring/drift-state-verifier.ts) - Runs every 10 minutes automatically - Checks closed trades (24h) vs actual Drift positions - Retries close if mismatch found - Sends Telegram alerts 2. Manual verification API (app/api/monitoring/verify-drift-state) - POST: Force immediate verification check - GET: Service status 3. Integrated into startup (lib/startup/init-position-manager.ts) - Auto-starts on container boot - First check after 2min, then every 10min STATUS: Build failing due to TypeScript compilation timeout Need to fix and deploy, then investigate WHY Position Manager stops. This addresses symptom (stuck positions) but not root cause (PM stopping).
471 lines
18 KiB
TypeScript
471 lines
18 KiB
TypeScript
/**
|
|
* Position Manager Startup Initialization
|
|
*
|
|
* Ensures Position Manager starts monitoring on bot startup
|
|
* This prevents orphaned trades when the bot restarts
|
|
*/
|
|
|
|
import { getInitializedPositionManager } from '../trading/position-manager'
|
|
import { logger } from '../utils/logger'
|
|
import { initializeDriftService } from '../drift/client'
|
|
import { getPrismaClient, createTrade } from '../database/trades'
|
|
import { getMarketConfig, getMergedConfig } from '../../config/trading'
|
|
import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker'
|
|
import { startStopHuntTracking } from '../trading/stop-hunt-tracker'
|
|
import { startSmartValidation } from '../trading/smart-validation-queue'
|
|
import { startDataCleanup } from '../maintenance/data-cleanup'
|
|
import { startDriftStateVerifier } from '../monitoring/drift-state-verifier'
|
|
import { logCriticalError } from '../utils/persistent-logger'
|
|
import { sendPositionClosedNotification } from '../notifications/telegram'
|
|
|
|
let initStarted = false
|
|
|
|
export async function initializePositionManagerOnStartup() {
|
|
if (initStarted) {
|
|
return
|
|
}
|
|
|
|
initStarted = true
|
|
|
|
logger.log('🚀 Initializing Position Manager on startup...')
|
|
|
|
try {
|
|
// CRITICAL (Dec 5, 2025): Start services FIRST, before validation
|
|
// Bug fix: validateOpenTrades() returns early if no trades, skipping services
|
|
// Services must run regardless of open trades count
|
|
|
|
// CRITICAL (Dec 2, 2025): Start data cleanup service for 4-week retention
|
|
// User directive: "we want to store the data for 4 weeks"
|
|
// Runs daily at 3 AM to delete MarketData records older than 28 days
|
|
console.log('🧹 Starting data cleanup service...')
|
|
startDataCleanup()
|
|
|
|
// Start blocked signal price tracking
|
|
console.log('🔬 Starting blocked signal price tracker...')
|
|
startBlockedSignalTracking()
|
|
|
|
// Start stop hunt revenge tracker
|
|
console.log('🎯 Starting stop hunt revenge tracker...')
|
|
await startStopHuntTracking()
|
|
|
|
// Start smart entry validation queue (Nov 30, 2025)
|
|
console.log('🧠 Starting smart entry validation system...')
|
|
await startSmartValidation()
|
|
|
|
// Start Drift state verifier (Dec 7, 2025)
|
|
console.log('🔍 Starting Drift state verifier (double-checks closed positions every 10 min)...')
|
|
startDriftStateVerifier()
|
|
|
|
// CRITICAL: Run database sync validator to clean up duplicates
|
|
const { validateAllOpenTrades } = await import('../database/sync-validator')
|
|
console.log('🔍 Running database sync validation before Position Manager init...')
|
|
const validationResult = await validateAllOpenTrades()
|
|
|
|
if (validationResult.ghosts > 0) {
|
|
logger.log(`✅ Cleaned up ${validationResult.ghosts} ghost/duplicate trades`)
|
|
}
|
|
|
|
// Then validate open trades against Drift positions
|
|
await validateOpenTrades()
|
|
|
|
// CRITICAL: Detect orphaned positions (on Drift but not in database)
|
|
await detectOrphanedPositions()
|
|
|
|
const manager = await getInitializedPositionManager()
|
|
const status = manager.getStatus()
|
|
|
|
console.log(`✅ Position Manager ready - ${status.activeTradesCount} active trades`)
|
|
|
|
if (status.activeTradesCount > 0) {
|
|
console.log(`📊 Monitoring: ${status.symbols.join(', ')}`)
|
|
}
|
|
} catch (error) {
|
|
console.error('❌ Failed to initialize Position Manager on startup:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validate that open trades in database match actual Drift positions
|
|
*
|
|
* CRITICAL FIX (Nov 14, 2025):
|
|
* - Also checks trades marked as "closed" in DB that might still be open on Drift
|
|
* - Happens when close transaction fails but bot marks it as closed anyway
|
|
* - Restores Position Manager tracking for these orphaned positions
|
|
*/
|
|
async function validateOpenTrades() {
|
|
try {
|
|
const prisma = getPrismaClient()
|
|
|
|
// Get both truly open trades AND recently "closed" trades (last 24h)
|
|
// Recently closed trades might still be open if close transaction failed
|
|
// TEMPORARILY REDUCED: Check only last 5 closed trades to avoid rate limiting on startup
|
|
const [openTrades, recentlyClosedTrades] = await Promise.all([
|
|
prisma.trade.findMany({
|
|
where: { status: 'open' },
|
|
orderBy: { entryTime: 'asc' }
|
|
}),
|
|
prisma.trade.findMany({
|
|
where: {
|
|
exitReason: { not: null },
|
|
exitTime: { gte: new Date(Date.now() - 6 * 60 * 60 * 1000) } // Last 6 hours (reduced from 24h)
|
|
},
|
|
orderBy: { exitTime: 'desc' },
|
|
take: 5 // Reduced from 20 to avoid rate limiting
|
|
})
|
|
])
|
|
|
|
const allTradesToCheck = [...openTrades, ...recentlyClosedTrades]
|
|
|
|
if (allTradesToCheck.length === 0) {
|
|
logger.log('✅ No open trades to validate')
|
|
return
|
|
}
|
|
|
|
logger.log(`🔍 Validating ${openTrades.length} open + ${recentlyClosedTrades.length} recently closed trades against Drift...`)
|
|
|
|
// CRITICAL: Group trades by symbol to handle multiple DB entries for same Drift position
|
|
// This prevents reopening old closed trades when only the most recent should be restored
|
|
const tradesBySymbol = new Map<string, any[]>()
|
|
for (const trade of allTradesToCheck) {
|
|
const existing = tradesBySymbol.get(trade.symbol) || []
|
|
existing.push(trade)
|
|
tradesBySymbol.set(trade.symbol, existing)
|
|
}
|
|
|
|
const driftService = await initializeDriftService()
|
|
const driftPositions = await driftService.getAllPositions() // Get all positions once
|
|
|
|
// Process each symbol's trades (keep only most recent if multiple exist)
|
|
for (const [symbol, trades] of tradesBySymbol) {
|
|
// Sort by creation time, newest first
|
|
trades.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
|
|
const mostRecentTrade = trades[0]
|
|
const olderTrades = trades.slice(1)
|
|
|
|
// Close any older trades BEFORE validating the most recent
|
|
for (const oldTrade of olderTrades) {
|
|
if (oldTrade.exitReason === null) {
|
|
logger.log(`🗑️ Closing duplicate old trade: ${oldTrade.id} (${symbol}, created ${oldTrade.createdAt.toISOString()})`)
|
|
await prisma.trade.update({
|
|
where: { id: oldTrade.id },
|
|
data: {
|
|
status: 'closed',
|
|
exitTime: new Date(),
|
|
exitReason: 'DUPLICATE_CLEANUP',
|
|
exitPrice: oldTrade.entryPrice,
|
|
realizedPnL: 0,
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Now validate only the most recent trade for this symbol
|
|
const trade = mostRecentTrade
|
|
|
|
try {
|
|
const marketConfig = getMarketConfig(trade.symbol)
|
|
|
|
// Find matching Drift position by symbol
|
|
const position = driftPositions.find(p => p.symbol === trade.symbol)
|
|
|
|
if (!position || position.size === 0) {
|
|
// No position on Drift
|
|
if (trade.status === 'open') {
|
|
logger.log(`⚠️ PHANTOM TRADE: ${trade.symbol} marked open in DB but not found on Drift`)
|
|
logger.log(` 🗑️ Auto-closing phantom trade...`)
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'closed',
|
|
exitTime: new Date(),
|
|
exitReason: 'PHANTOM_TRADE_CLEANUP',
|
|
exitPrice: trade.entryPrice,
|
|
realizedPnL: 0,
|
|
realizedPnLPercent: 0,
|
|
}
|
|
})
|
|
}
|
|
// If already closed in DB and not on Drift, that's correct - skip
|
|
continue
|
|
}
|
|
|
|
// Position EXISTS on Drift
|
|
const driftDirection = position.side.toLowerCase() as 'long' | 'short'
|
|
|
|
if (driftDirection !== trade.direction) {
|
|
logger.log(`⚠️ DIRECTION MISMATCH: ${trade.symbol} DB=${trade.direction} Drift=${driftDirection}`)
|
|
continue
|
|
}
|
|
|
|
// CRITICAL: If DB says closed but Drift shows open, restore tracking!
|
|
if (trade.exitReason !== null) {
|
|
logger.log(`🔴 CRITICAL: ${trade.symbol} marked as CLOSED in DB but still OPEN on Drift!`)
|
|
logger.log(` DB entry: $${trade.entryPrice.toFixed(2)} | Drift entry: $${position.entryPrice.toFixed(2)}`)
|
|
logger.log(` DB exit: ${trade.exitReason} at ${trade.exitTime?.toISOString()}`)
|
|
logger.log(` Drift: ${position.size} ${trade.symbol} ${driftDirection} @ $${position.entryPrice.toFixed(2)}`)
|
|
logger.log(` 🔄 Reopening trade and correcting entry price to match Drift...`)
|
|
|
|
// Calculate position size in USD using Drift's entry price
|
|
const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex)
|
|
const positionSizeUSD = position.size * currentPrice
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'open',
|
|
exitReason: null,
|
|
exitTime: null,
|
|
exitPrice: null,
|
|
entryPrice: position.entryPrice, // CRITICAL: Use Drift's actual entry price, not DB value
|
|
positionSizeUSD: positionSizeUSD, // Update to current size (may be runner after TP1)
|
|
// Keep original realizedPnL from partial closes if any
|
|
}
|
|
})
|
|
|
|
logger.log(` ✅ Trade restored with corrected entry: $${position.entryPrice.toFixed(2)} (was $${trade.entryPrice.toFixed(2)})`)
|
|
} else {
|
|
logger.log(`✅ ${trade.symbol} ${trade.direction}: Position verified on Drift`)
|
|
}
|
|
|
|
// CRITICAL FIX (Nov 16, 2025): Restore missing on-chain orders
|
|
// Ghost position closed at 22:03 because orders were missing after validator cleanup
|
|
// This ensures EVERY verified position has on-chain TP/SL protection
|
|
if (position && Math.abs(position.size) >= 0.01) {
|
|
await restoreOrdersIfMissing(trade, position, driftService, prisma)
|
|
}
|
|
|
|
} catch (posError) {
|
|
console.error(`❌ Error validating trade ${trade.symbol}:`, posError)
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('❌ Error in validateOpenTrades:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore on-chain exit orders if missing (Nov 16, 2025)
|
|
*
|
|
* CRITICAL: After validator cleanups or container restarts, positions may exist
|
|
* on Drift without any on-chain TP/SL orders. This leaves only Position Manager
|
|
* software protection - if bot crashes, position is completely unprotected.
|
|
*
|
|
* This function checks if orders exist and places them if missing.
|
|
*/
|
|
async function restoreOrdersIfMissing(
|
|
trade: any,
|
|
position: any,
|
|
driftService: any,
|
|
prisma: any
|
|
): Promise<void> {
|
|
try {
|
|
// Check if position has any reduce-only orders
|
|
const hasOrders = position.orders && position.orders.length > 0
|
|
|
|
if (hasOrders) {
|
|
logger.log(`✅ ${trade.symbol} has ${position.orders.length} on-chain orders - protection active`)
|
|
return // Orders exist, nothing to do
|
|
}
|
|
|
|
logger.log(`⚠️ ${trade.symbol} has NO on-chain orders - restoring TP/SL protection...`)
|
|
|
|
// Import order placement function
|
|
const { placeExitOrders } = await import('../drift/orders')
|
|
|
|
// Place exit orders using trade's TP/SL prices
|
|
const result = await placeExitOrders({
|
|
symbol: trade.symbol,
|
|
direction: trade.direction,
|
|
entryPrice: trade.entryPrice,
|
|
tp1Price: trade.takeProfit1Price,
|
|
tp2Price: trade.takeProfit2Price,
|
|
stopLossPrice: trade.stopLossPrice,
|
|
positionSizeUSD: trade.positionSizeUSD,
|
|
tp1SizePercent: 75,
|
|
tp2SizePercent: 0, // TP2-as-runner
|
|
})
|
|
|
|
if (result.success) {
|
|
logger.log(`✅ Orders restored for ${trade.symbol}:`)
|
|
logger.log(` TP1: $${trade.takeProfit1Price.toFixed(4)} (75%)`)
|
|
logger.log(` TP2: $${trade.takeProfit2Price.toFixed(4)} (runner trigger)`)
|
|
logger.log(` SL: $${trade.stopLossPrice.toFixed(4)}`)
|
|
logger.log(` TX: ${result.signatures?.[0]?.slice(0, 8)}...`)
|
|
|
|
// Update database with order transaction signatures
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
tp1OrderTx: result.signatures?.[0],
|
|
tp2OrderTx: result.signatures?.[1],
|
|
slOrderTx: result.signatures?.[2],
|
|
}
|
|
})
|
|
} else {
|
|
console.error(`❌ Failed to restore orders for ${trade.symbol}:`, result.error)
|
|
console.error(` 🚨 CRITICAL: Position is unprotected - only Position Manager monitoring active`)
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error(`❌ Error restoring orders for ${trade.symbol}:`, error)
|
|
console.error(` 🚨 CRITICAL: Position may be unprotected`)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect orphaned positions - positions on Drift with NO database record
|
|
*
|
|
* CRITICAL FIX (Nov 21, 2025): Prevents ghost positions from database save failures
|
|
*
|
|
* This can happen when:
|
|
* - Database save fails silently during trade execution
|
|
* - Prisma transaction rolls back but no error thrown
|
|
* - Container restart interrupts database save
|
|
*
|
|
* Recovery:
|
|
* 1. Create retroactive database record with current Drift data
|
|
* 2. Send Telegram alert about orphaned position found
|
|
* 3. Add to Position Manager for normal TP/SL monitoring
|
|
*/
|
|
async function detectOrphanedPositions() {
|
|
try {
|
|
const prisma = getPrismaClient()
|
|
const driftService = await initializeDriftService()
|
|
|
|
logger.log('🔍 Checking for orphaned positions on Drift...')
|
|
|
|
// Get all open positions from Drift
|
|
const driftPositions = await driftService.getAllPositions()
|
|
|
|
if (driftPositions.length === 0) {
|
|
logger.log('✅ No positions on Drift')
|
|
return
|
|
}
|
|
|
|
logger.log(`🔍 Found ${driftPositions.length} positions on Drift, checking database...`)
|
|
|
|
// Get all open trades from database
|
|
const openTrades = await prisma.trade.findMany({
|
|
where: { status: 'open' },
|
|
select: { symbol: true, positionId: true, direction: true }
|
|
})
|
|
|
|
const trackedSymbols = new Set(openTrades.map(t => `${t.symbol}-${t.direction}`))
|
|
|
|
// Check each Drift position
|
|
for (const position of driftPositions) {
|
|
const positionKey = `${position.symbol}-${position.side.toLowerCase()}`
|
|
|
|
if (trackedSymbols.has(positionKey)) {
|
|
logger.log(`✅ ${position.symbol} ${position.side} tracked in database`)
|
|
continue
|
|
}
|
|
|
|
// ORPHAN DETECTED!
|
|
// Get current price from Drift oracle
|
|
const marketConfig = getMarketConfig(position.symbol)
|
|
const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex)
|
|
const positionSizeUSD = Math.abs(position.size) * currentPrice
|
|
|
|
logger.log(`🚨 ORPHAN POSITION DETECTED!`)
|
|
logger.log(` Symbol: ${position.symbol}`)
|
|
logger.log(` Direction: ${position.side}`)
|
|
logger.log(` Size: ${Math.abs(position.size)} (notional: $${positionSizeUSD.toFixed(2)})`)
|
|
logger.log(` Entry: $${position.entryPrice.toFixed(4)}`)
|
|
logger.log(` Current: $${currentPrice.toFixed(4)}`)
|
|
|
|
// Log to persistent file
|
|
logCriticalError('ORPHAN POSITION DETECTED - Creating retroactive database record', {
|
|
symbol: position.symbol,
|
|
direction: position.side.toLowerCase(),
|
|
entryPrice: position.entryPrice,
|
|
size: Math.abs(position.size),
|
|
currentPrice: currentPrice,
|
|
detectedAt: new Date().toISOString()
|
|
})
|
|
|
|
try {
|
|
// Get config for TP/SL calculation
|
|
const config = getMergedConfig()
|
|
|
|
// Calculate estimated TP/SL prices based on current config
|
|
const direction = position.side.toLowerCase() as 'long' | 'short'
|
|
const entryPrice = position.entryPrice
|
|
|
|
// Calculate TP/SL using same logic as execute endpoint
|
|
const stopLossPrice = direction === 'long'
|
|
? entryPrice * (1 + config.stopLossPercent / 100)
|
|
: entryPrice * (1 - config.stopLossPercent / 100)
|
|
|
|
const tp1Price = direction === 'long'
|
|
? entryPrice * (1 + config.takeProfit1Percent / 100)
|
|
: entryPrice * (1 - config.takeProfit1Percent / 100)
|
|
|
|
const tp2Price = direction === 'long'
|
|
? entryPrice * (1 + config.takeProfit2Percent / 100)
|
|
: entryPrice * (1 - config.takeProfit2Percent / 100)
|
|
|
|
// Create retroactive database record
|
|
logger.log(`🔄 Creating retroactive database record...`)
|
|
const trade = await createTrade({
|
|
positionId: `ORPHAN-${Date.now()}`, // Fake position ID since we don't have transaction
|
|
symbol: position.symbol,
|
|
direction: direction,
|
|
entryPrice: entryPrice,
|
|
positionSizeUSD: positionSizeUSD,
|
|
leverage: config.leverage,
|
|
stopLossPrice: stopLossPrice,
|
|
takeProfit1Price: tp1Price,
|
|
takeProfit2Price: tp2Price,
|
|
tp1SizePercent: config.takeProfit1SizePercent || 75,
|
|
tp2SizePercent: config.takeProfit2SizePercent || 0,
|
|
entryOrderTx: `ORPHAN-${Date.now()}`, // Fake transaction ID
|
|
configSnapshot: config,
|
|
signalSource: 'orphan_recovery',
|
|
timeframe: 'unknown',
|
|
status: 'open',
|
|
})
|
|
|
|
logger.log(`✅ Retroactive database record created: ${trade.id}`)
|
|
|
|
// Send Telegram notification
|
|
try {
|
|
await sendPositionClosedNotification({
|
|
symbol: position.symbol,
|
|
direction: direction,
|
|
entryPrice: entryPrice,
|
|
exitPrice: currentPrice,
|
|
positionSize: positionSizeUSD,
|
|
realizedPnL: 0, // Unknown
|
|
holdTimeSeconds: 0,
|
|
exitReason: 'ORPHAN_DETECTED',
|
|
maxGain: 0,
|
|
maxDrawdown: 0,
|
|
})
|
|
} catch (telegramError) {
|
|
console.error('Failed to send orphan notification:', telegramError)
|
|
}
|
|
|
|
logger.log(`🎯 Orphan position now tracked and monitored`)
|
|
|
|
} catch (recoveryError) {
|
|
console.error(`❌ Failed to recover orphan position ${position.symbol}:`, recoveryError)
|
|
logCriticalError('ORPHAN RECOVERY FAILED', {
|
|
symbol: position.symbol,
|
|
error: recoveryError instanceof Error ? recoveryError.message : String(recoveryError)
|
|
})
|
|
}
|
|
}
|
|
|
|
logger.log('✅ Orphan position detection complete')
|
|
|
|
} catch (error) {
|
|
console.error('❌ Error detecting orphaned positions:', error)
|
|
logCriticalError('Orphan detection failed', {
|
|
error: error instanceof Error ? error.message : String(error)
|
|
})
|
|
}
|
|
}
|