feat: Add comprehensive database save protection system
INVESTIGATION RESULT: No database failure occurred - trade was saved correctly. However, implemented 5-layer protection against future failures: 1. Persistent File Logger (lib/utils/persistent-logger.ts) - Survives container restarts - Logs to /app/logs/errors.log - Daily rotation, 30-day retention 2. Database Save Retry Logic (lib/database/trades.ts) - 3 retry attempts with exponential backoff (1s, 2s, 4s) - Immediate verification query after each create - Persistent logging of all attempts 3. Orphan Position Detection (lib/startup/init-position-manager.ts) - Runs on every container startup - Queries Drift for positions without database records - Creates retroactive Trade records - Sends Telegram alerts - Restores Position Manager monitoring 4. Critical Logging (app/api/trading/execute/route.ts) - Database failures logged with full trade details - Stack traces preserved for debugging 5. Infrastructure (logs directory + Docker volume) - Mounted at /home/icke/traderv4/logs - Configured in docker-compose.yml Trade from Nov 21 00:40:14 CET: - Found in database: cmi82qg590001tn079c3qpw4r - SHORT SOL-PERP 33.69 → 34.67 SL - P&L: -9.17 - Closed at 01:17:03 CET (37 minutes duration) - No database failure occurred Future Protection: - Retry logic catches transient failures - Verification prevents silent failures - Orphan detection catches anything missed - Persistent logs enable post-mortem analysis - System now bulletproof for 16 → 00k journey
This commit is contained in:
mindesbunister
@@ -7,10 +7,12 @@
|
||||
|
||||
import { getInitializedPositionManager } from '../trading/position-manager'
|
||||
import { initializeDriftService } from '../drift/client'
|
||||
import { getPrismaClient } from '../database/trades'
|
||||
import { getMarketConfig } from '../../config/trading'
|
||||
import { getPrismaClient, createTrade } from '../database/trades'
|
||||
import { getMarketConfig, getMergedConfig } from '../../config/trading'
|
||||
import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker'
|
||||
import { startStopHuntTracking } from '../trading/stop-hunt-tracker'
|
||||
import { logCriticalError } from '../utils/persistent-logger'
|
||||
import { sendPositionClosedNotification } from '../notifications/telegram'
|
||||
|
||||
let initStarted = false
|
||||
|
||||
@@ -36,6 +38,9 @@ export async function initializePositionManagerOnStartup() {
|
||||
// Then validate open trades against Drift positions
|
||||
await validateOpenTrades()
|
||||
|
||||
// CRITICAL: Detect orphaned positions (on Drift but not in database)
|
||||
await detectOrphanedPositions()
|
||||
|
||||
const manager = await getInitializedPositionManager()
|
||||
const status = manager.getStatus()
|
||||
|
||||
@@ -286,3 +291,158 @@ async function restoreOrdersIfMissing(
|
||||
console.error(` 🚨 CRITICAL: Position may be unprotected`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect orphaned positions - positions on Drift with NO database record
|
||||
*
|
||||
* CRITICAL FIX (Nov 21, 2025): Prevents ghost positions from database save failures
|
||||
*
|
||||
* This can happen when:
|
||||
* - Database save fails silently during trade execution
|
||||
* - Prisma transaction rolls back but no error thrown
|
||||
* - Container restart interrupts database save
|
||||
*
|
||||
* Recovery:
|
||||
* 1. Create retroactive database record with current Drift data
|
||||
* 2. Send Telegram alert about orphaned position found
|
||||
* 3. Add to Position Manager for normal TP/SL monitoring
|
||||
*/
|
||||
async function detectOrphanedPositions() {
|
||||
try {
|
||||
const prisma = getPrismaClient()
|
||||
const driftService = await initializeDriftService()
|
||||
|
||||
console.log('🔍 Checking for orphaned positions on Drift...')
|
||||
|
||||
// Get all open positions from Drift
|
||||
const driftPositions = await driftService.getAllPositions()
|
||||
|
||||
if (driftPositions.length === 0) {
|
||||
console.log('✅ No positions on Drift')
|
||||
return
|
||||
}
|
||||
|
||||
console.log(`🔍 Found ${driftPositions.length} positions on Drift, checking database...`)
|
||||
|
||||
// Get all open trades from database
|
||||
const openTrades = await prisma.trade.findMany({
|
||||
where: { status: 'open' },
|
||||
select: { symbol: true, positionId: true, direction: true }
|
||||
})
|
||||
|
||||
const trackedSymbols = new Set(openTrades.map(t => `${t.symbol}-${t.direction}`))
|
||||
|
||||
// Check each Drift position
|
||||
for (const position of driftPositions) {
|
||||
const positionKey = `${position.symbol}-${position.side.toLowerCase()}`
|
||||
|
||||
if (trackedSymbols.has(positionKey)) {
|
||||
console.log(`✅ ${position.symbol} ${position.side} tracked in database`)
|
||||
continue
|
||||
}
|
||||
|
||||
// ORPHAN DETECTED!
|
||||
// Get current price from Drift oracle
|
||||
const marketConfig = getMarketConfig(position.symbol)
|
||||
const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex)
|
||||
const positionSizeUSD = Math.abs(position.size) * currentPrice
|
||||
|
||||
console.log(`🚨 ORPHAN POSITION DETECTED!`)
|
||||
console.log(` Symbol: ${position.symbol}`)
|
||||
console.log(` Direction: ${position.side}`)
|
||||
console.log(` Size: ${Math.abs(position.size)} (notional: $${positionSizeUSD.toFixed(2)})`)
|
||||
console.log(` Entry: $${position.entryPrice.toFixed(4)}`)
|
||||
console.log(` Current: $${currentPrice.toFixed(4)}`)
|
||||
|
||||
// Log to persistent file
|
||||
logCriticalError('ORPHAN POSITION DETECTED - Creating retroactive database record', {
|
||||
symbol: position.symbol,
|
||||
direction: position.side.toLowerCase(),
|
||||
entryPrice: position.entryPrice,
|
||||
size: Math.abs(position.size),
|
||||
currentPrice: currentPrice,
|
||||
detectedAt: new Date().toISOString()
|
||||
})
|
||||
|
||||
try {
|
||||
// Get config for TP/SL calculation
|
||||
const config = getMergedConfig()
|
||||
|
||||
// Calculate estimated TP/SL prices based on current config
|
||||
const direction = position.side.toLowerCase() as 'long' | 'short'
|
||||
const entryPrice = position.entryPrice
|
||||
|
||||
// Calculate TP/SL using same logic as execute endpoint
|
||||
const stopLossPrice = direction === 'long'
|
||||
? entryPrice * (1 + config.stopLossPercent / 100)
|
||||
: entryPrice * (1 - config.stopLossPercent / 100)
|
||||
|
||||
const tp1Price = direction === 'long'
|
||||
? entryPrice * (1 + config.takeProfit1Percent / 100)
|
||||
: entryPrice * (1 - config.takeProfit1Percent / 100)
|
||||
|
||||
const tp2Price = direction === 'long'
|
||||
? entryPrice * (1 + config.takeProfit2Percent / 100)
|
||||
: entryPrice * (1 - config.takeProfit2Percent / 100)
|
||||
|
||||
// Create retroactive database record
|
||||
console.log(`🔄 Creating retroactive database record...`)
|
||||
const trade = await createTrade({
|
||||
positionId: `ORPHAN-${Date.now()}`, // Fake position ID since we don't have transaction
|
||||
symbol: position.symbol,
|
||||
direction: direction,
|
||||
entryPrice: entryPrice,
|
||||
positionSizeUSD: positionSizeUSD,
|
||||
leverage: config.leverage,
|
||||
stopLossPrice: stopLossPrice,
|
||||
takeProfit1Price: tp1Price,
|
||||
takeProfit2Price: tp2Price,
|
||||
tp1SizePercent: config.takeProfit1SizePercent || 75,
|
||||
tp2SizePercent: config.takeProfit2SizePercent || 0,
|
||||
entryOrderTx: `ORPHAN-${Date.now()}`, // Fake transaction ID
|
||||
configSnapshot: config,
|
||||
signalSource: 'orphan_recovery',
|
||||
timeframe: 'unknown',
|
||||
status: 'open',
|
||||
})
|
||||
|
||||
console.log(`✅ Retroactive database record created: ${trade.id}`)
|
||||
|
||||
// Send Telegram notification
|
||||
try {
|
||||
await sendPositionClosedNotification({
|
||||
symbol: position.symbol,
|
||||
direction: direction,
|
||||
entryPrice: entryPrice,
|
||||
exitPrice: currentPrice,
|
||||
positionSize: positionSizeUSD,
|
||||
realizedPnL: 0, // Unknown
|
||||
holdTimeSeconds: 0,
|
||||
exitReason: 'ORPHAN_DETECTED',
|
||||
maxGain: 0,
|
||||
maxDrawdown: 0,
|
||||
})
|
||||
} catch (telegramError) {
|
||||
console.error('Failed to send orphan notification:', telegramError)
|
||||
}
|
||||
|
||||
console.log(`🎯 Orphan position now tracked and monitored`)
|
||||
|
||||
} catch (recoveryError) {
|
||||
console.error(`❌ Failed to recover orphan position ${position.symbol}:`, recoveryError)
|
||||
logCriticalError('ORPHAN RECOVERY FAILED', {
|
||||
symbol: position.symbol,
|
||||
error: recoveryError instanceof Error ? recoveryError.message : String(recoveryError)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
console.log('✅ Orphan position detection complete')
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error detecting orphaned positions:', error)
|
||||
logCriticalError('Orphan detection failed', {
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user