**Problem 1: Rate Limit Cascade** - Position Manager tried to close repeatedly, overwhelming Helius RPC (10 req/s limit) - Base retry delay was too aggressive (2s → 4s → 8s) - No graceful handling when 429 errors occur **Problem 2: Orphaned Positions After Restart** - Container restarts lost Position Manager state - Positions marked 'closed' in DB but still open on Drift (failed close transactions) - No cross-validation between database and actual Drift positions **Solutions Implemented:** 1. **Increased retry delays (orders.ts)**: - Base delay: 2s → 5s (progression now 5s → 10s → 20s) - Reduces RPC pressure during rate limit situations - Gives Helius time to recover between retries - Documented Helius limits: 100 req/s burst, 10 req/s sustained (free tier) 2. **Startup position validation (init-position-manager.ts)**: - Cross-checks last 24h of 'closed' trades against actual Drift positions - If DB says closed but Drift shows open → reopens in DB to restore tracking - Prevents unmonitored positions from existing after container restarts - Logs detailed mismatch info for debugging 3. **Rate limit-aware exit handling (position-manager.ts)**: - Detects 429 errors during position close - Keeps trade in monitoring instead of removing it - Natural retry on next price update (vs aggressive 2s loop) - Prevents marking position as closed when transaction actually failed **Impact:** - Eliminates orphaned positions after restarts - Reduces RPC pressure by 2.5x (5s vs 2s base delay) - Graceful degradation under rate limits - Position Manager continues monitoring even during temporary RPC issues **Testing needed:** - Monitor next container restart to verify position restoration works - Check rate limit analytics after next close attempt - Verify no more phantom 'closed' positions when Drift shows open
151 lines
5.3 KiB
TypeScript
151 lines
5.3 KiB
TypeScript
/**
|
|
* Position Manager Startup Initialization
|
|
*
|
|
* Ensures Position Manager starts monitoring on bot startup
|
|
* This prevents orphaned trades when the bot restarts
|
|
*/
|
|
|
|
import { getInitializedPositionManager } from '../trading/position-manager'
|
|
import { initializeDriftService } from '../drift/client'
|
|
import { getPrismaClient } from '../database/trades'
|
|
import { getMarketConfig } from '../../config/trading'
|
|
|
|
let initStarted = false
|
|
|
|
export async function initializePositionManagerOnStartup() {
|
|
if (initStarted) {
|
|
return
|
|
}
|
|
|
|
initStarted = true
|
|
|
|
console.log('🚀 Initializing Position Manager on startup...')
|
|
|
|
try {
|
|
// Validate open trades against Drift positions BEFORE starting Position Manager
|
|
await validateOpenTrades()
|
|
|
|
const manager = await getInitializedPositionManager()
|
|
const status = manager.getStatus()
|
|
|
|
console.log(`✅ Position Manager ready - ${status.activeTradesCount} active trades`)
|
|
|
|
if (status.activeTradesCount > 0) {
|
|
console.log(`📊 Monitoring: ${status.symbols.join(', ')}`)
|
|
}
|
|
} catch (error) {
|
|
console.error('❌ Failed to initialize Position Manager on startup:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validate that open trades in database match actual Drift positions
|
|
*
|
|
* CRITICAL FIX (Nov 14, 2025):
|
|
* - Also checks trades marked as "closed" in DB that might still be open on Drift
|
|
* - Happens when close transaction fails but bot marks it as closed anyway
|
|
* - Restores Position Manager tracking for these orphaned positions
|
|
*/
|
|
async function validateOpenTrades() {
|
|
try {
|
|
const prisma = getPrismaClient()
|
|
|
|
// Get both truly open trades AND recently "closed" trades (last 24h)
|
|
// Recently closed trades might still be open if close transaction failed
|
|
const [openTrades, recentlyClosedTrades] = await Promise.all([
|
|
prisma.trade.findMany({
|
|
where: { status: 'open' },
|
|
orderBy: { entryTime: 'asc' }
|
|
}),
|
|
prisma.trade.findMany({
|
|
where: {
|
|
exitReason: { not: null },
|
|
exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } // Last 24 hours
|
|
},
|
|
orderBy: { exitTime: 'desc' },
|
|
take: 20 // Check last 20 closed trades
|
|
})
|
|
])
|
|
|
|
const allTradesToCheck = [...openTrades, ...recentlyClosedTrades]
|
|
|
|
if (allTradesToCheck.length === 0) {
|
|
console.log('✅ No open trades to validate')
|
|
return
|
|
}
|
|
|
|
console.log(`🔍 Validating ${openTrades.length} open + ${recentlyClosedTrades.length} recently closed trades against Drift...`)
|
|
|
|
const driftService = await initializeDriftService()
|
|
const driftPositions = await driftService.getAllPositions() // Get all positions once
|
|
|
|
for (const trade of allTradesToCheck) {
|
|
try {
|
|
const marketConfig = getMarketConfig(trade.symbol)
|
|
|
|
// Find matching Drift position by symbol
|
|
const position = driftPositions.find(p => p.symbol === trade.symbol)
|
|
|
|
if (!position || position.size === 0) {
|
|
// No position on Drift
|
|
if (trade.status === 'open') {
|
|
console.log(`⚠️ PHANTOM TRADE: ${trade.symbol} marked open in DB but not found on Drift`)
|
|
console.log(` 🗑️ Auto-closing phantom trade...`)
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'closed',
|
|
exitTime: new Date(),
|
|
exitReason: 'PHANTOM_TRADE_CLEANUP',
|
|
exitPrice: trade.entryPrice,
|
|
realizedPnL: 0,
|
|
realizedPnLPercent: 0,
|
|
}
|
|
})
|
|
}
|
|
// If already closed in DB and not on Drift, that's correct - skip
|
|
continue
|
|
}
|
|
|
|
// Position EXISTS on Drift
|
|
const driftDirection = position.direction.toLowerCase() as 'long' | 'short'
|
|
|
|
if (driftDirection !== trade.direction) {
|
|
console.log(`⚠️ DIRECTION MISMATCH: ${trade.symbol} DB=${trade.direction} Drift=${driftDirection}`)
|
|
continue
|
|
}
|
|
|
|
// CRITICAL: If DB says closed but Drift shows open, restore tracking!
|
|
if (trade.exitReason !== null) {
|
|
console.log(`🔴 CRITICAL: ${trade.symbol} marked as CLOSED in DB but still OPEN on Drift!`)
|
|
console.log(` DB exit: ${trade.exitReason} at ${trade.exitTime?.toISOString()}`)
|
|
console.log(` Drift: ${position.size} ${trade.symbol} ${driftDirection} @ $${position.entryPrice.toFixed(2)}`)
|
|
console.log(` 🔄 Reopening trade in DB to restore Position Manager tracking...`)
|
|
|
|
await prisma.trade.update({
|
|
where: { id: trade.id },
|
|
data: {
|
|
status: 'open',
|
|
exitReason: null,
|
|
exitTime: null,
|
|
exitPrice: null,
|
|
// Keep original realizedPnL from partial closes if any
|
|
}
|
|
})
|
|
|
|
console.log(` ✅ Trade restored - Position Manager will now monitor it`)
|
|
} else {
|
|
console.log(`✅ ${trade.symbol} ${trade.direction}: Position verified on Drift`)
|
|
}
|
|
|
|
} catch (posError) {
|
|
console.error(`❌ Error validating trade ${trade.symbol}:`, posError)
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('❌ Error in validateOpenTrades:', error)
|
|
}
|
|
}
|