Files
trading_bot_v4/lib/startup/init-position-manager.ts
mindesbunister 27eb5d4fe8 fix: Critical rate limit handling + startup position restoration
**Problem 1: Rate Limit Cascade**
- Position Manager tried to close repeatedly, overwhelming Helius RPC (10 req/s limit)
- Base retry delay was too aggressive (2s → 4s → 8s)
- No graceful handling when 429 errors occur

**Problem 2: Orphaned Positions After Restart**
- Container restarts lost Position Manager state
- Positions marked 'closed' in DB but still open on Drift (failed close transactions)
- No cross-validation between database and actual Drift positions

**Solutions Implemented:**

1. **Increased retry delays (orders.ts)**:
   - Base delay: 2s → 5s (progression now 5s → 10s → 20s)
   - Reduces RPC pressure during rate limit situations
   - Gives Helius time to recover between retries
   - Documented Helius limits: 100 req/s burst, 10 req/s sustained (free tier)

2. **Startup position validation (init-position-manager.ts)**:
   - Cross-checks last 24h of 'closed' trades against actual Drift positions
   - If DB says closed but Drift shows open → reopens in DB to restore tracking
   - Prevents unmonitored positions from existing after container restarts
   - Logs detailed mismatch info for debugging

3. **Rate limit-aware exit handling (position-manager.ts)**:
   - Detects 429 errors during position close
   - Keeps trade in monitoring instead of removing it
   - Natural retry on next price update (vs aggressive 2s loop)
   - Prevents marking position as closed when transaction actually failed

**Impact:**
- Eliminates orphaned positions after restarts
- Reduces RPC pressure by 2.5x (5s vs 2s base delay)
- Graceful degradation under rate limits
- Position Manager continues monitoring even during temporary RPC issues

**Testing needed:**
- Monitor next container restart to verify position restoration works
- Check rate limit analytics after next close attempt
- Verify no more phantom 'closed' positions when Drift shows open
2025-11-14 09:50:13 +01:00

151 lines
5.3 KiB
TypeScript

/**
* Position Manager Startup Initialization
*
* Ensures Position Manager starts monitoring on bot startup
* This prevents orphaned trades when the bot restarts
*/
import { getInitializedPositionManager } from '../trading/position-manager'
import { initializeDriftService } from '../drift/client'
import { getPrismaClient } from '../database/trades'
import { getMarketConfig } from '../../config/trading'
let initStarted = false
export async function initializePositionManagerOnStartup() {
if (initStarted) {
return
}
initStarted = true
console.log('🚀 Initializing Position Manager on startup...')
try {
// Validate open trades against Drift positions BEFORE starting Position Manager
await validateOpenTrades()
const manager = await getInitializedPositionManager()
const status = manager.getStatus()
console.log(`✅ Position Manager ready - ${status.activeTradesCount} active trades`)
if (status.activeTradesCount > 0) {
console.log(`📊 Monitoring: ${status.symbols.join(', ')}`)
}
} catch (error) {
console.error('❌ Failed to initialize Position Manager on startup:', error)
}
}
/**
* Validate that open trades in database match actual Drift positions
*
* CRITICAL FIX (Nov 14, 2025):
* - Also checks trades marked as "closed" in DB that might still be open on Drift
* - Happens when close transaction fails but bot marks it as closed anyway
* - Restores Position Manager tracking for these orphaned positions
*/
async function validateOpenTrades() {
try {
const prisma = getPrismaClient()
// Get both truly open trades AND recently "closed" trades (last 24h)
// Recently closed trades might still be open if close transaction failed
const [openTrades, recentlyClosedTrades] = await Promise.all([
prisma.trade.findMany({
where: { status: 'open' },
orderBy: { entryTime: 'asc' }
}),
prisma.trade.findMany({
where: {
exitReason: { not: null },
exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } // Last 24 hours
},
orderBy: { exitTime: 'desc' },
take: 20 // Check last 20 closed trades
})
])
const allTradesToCheck = [...openTrades, ...recentlyClosedTrades]
if (allTradesToCheck.length === 0) {
console.log('✅ No open trades to validate')
return
}
console.log(`🔍 Validating ${openTrades.length} open + ${recentlyClosedTrades.length} recently closed trades against Drift...`)
const driftService = await initializeDriftService()
const driftPositions = await driftService.getAllPositions() // Get all positions once
for (const trade of allTradesToCheck) {
try {
const marketConfig = getMarketConfig(trade.symbol)
// Find matching Drift position by symbol
const position = driftPositions.find(p => p.symbol === trade.symbol)
if (!position || position.size === 0) {
// No position on Drift
if (trade.status === 'open') {
console.log(`⚠️ PHANTOM TRADE: ${trade.symbol} marked open in DB but not found on Drift`)
console.log(` 🗑️ Auto-closing phantom trade...`)
await prisma.trade.update({
where: { id: trade.id },
data: {
status: 'closed',
exitTime: new Date(),
exitReason: 'PHANTOM_TRADE_CLEANUP',
exitPrice: trade.entryPrice,
realizedPnL: 0,
realizedPnLPercent: 0,
}
})
}
// If already closed in DB and not on Drift, that's correct - skip
continue
}
// Position EXISTS on Drift
const driftDirection = position.direction.toLowerCase() as 'long' | 'short'
if (driftDirection !== trade.direction) {
console.log(`⚠️ DIRECTION MISMATCH: ${trade.symbol} DB=${trade.direction} Drift=${driftDirection}`)
continue
}
// CRITICAL: If DB says closed but Drift shows open, restore tracking!
if (trade.exitReason !== null) {
console.log(`🔴 CRITICAL: ${trade.symbol} marked as CLOSED in DB but still OPEN on Drift!`)
console.log(` DB exit: ${trade.exitReason} at ${trade.exitTime?.toISOString()}`)
console.log(` Drift: ${position.size} ${trade.symbol} ${driftDirection} @ $${position.entryPrice.toFixed(2)}`)
console.log(` 🔄 Reopening trade in DB to restore Position Manager tracking...`)
await prisma.trade.update({
where: { id: trade.id },
data: {
status: 'open',
exitReason: null,
exitTime: null,
exitPrice: null,
// Keep original realizedPnL from partial closes if any
}
})
console.log(` ✅ Trade restored - Position Manager will now monitor it`)
} else {
console.log(`${trade.symbol} ${trade.direction}: Position verified on Drift`)
}
} catch (posError) {
console.error(`❌ Error validating trade ${trade.symbol}:`, posError)
}
}
} catch (error) {
console.error('❌ Error in validateOpenTrades:', error)
}
}