feat: Add comprehensive database save protection system
INVESTIGATION RESULT: No database failure occurred - trade was saved correctly. However, implemented 5-layer protection against future failures: 1. Persistent File Logger (lib/utils/persistent-logger.ts) - Survives container restarts - Logs to /app/logs/errors.log - Daily rotation, 30-day retention 2. Database Save Retry Logic (lib/database/trades.ts) - 3 retry attempts with exponential backoff (1s, 2s, 4s) - Immediate verification query after each create - Persistent logging of all attempts 3. Orphan Position Detection (lib/startup/init-position-manager.ts) - Runs on every container startup - Queries Drift for positions without database records - Creates retroactive Trade records - Sends Telegram alerts - Restores Position Manager monitoring 4. Critical Logging (app/api/trading/execute/route.ts) - Database failures logged with full trade details - Stack traces preserved for debugging 5. Infrastructure (logs directory + Docker volume) - Mounted at /home/icke/traderv4/logs - Configured in docker-compose.yml Trade from Nov 21 00:40:14 CET: - Found in database: cmi82qg590001tn079c3qpw4r - SHORT SOL-PERP 33.69 → 34.67 SL - P&L: -9.17 - Closed at 01:17:03 CET (37 minutes duration) - No database failure occurred Future Protection: - Retry logic catches transient failures - Verification prevents silent failures - Orphan detection catches anything missed - Persistent logs enable post-mortem analysis - System now bulletproof for 16 → 00k journey
This commit is contained in:
mindesbunister
@@ -3,6 +3,7 @@
|
||||
*/
|
||||
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { logCriticalError, logDatabaseOperation } from '../utils/persistent-logger'
|
||||
|
||||
// Singleton Prisma client
|
||||
let prisma: PrismaClient | null = null
|
||||
@@ -97,68 +98,135 @@ export interface UpdateTradeExitParams {
|
||||
export async function createTrade(params: CreateTradeParams) {
|
||||
const prisma = getPrismaClient()
|
||||
|
||||
try {
|
||||
// Calculate entry slippage if expected price provided
|
||||
let entrySlippagePct: number | undefined
|
||||
if (params.expectedEntryPrice && params.entrySlippage !== undefined) {
|
||||
entrySlippagePct = params.entrySlippage
|
||||
}
|
||||
|
||||
const trade = await prisma.trade.create({
|
||||
data: {
|
||||
// Retry logic with exponential backoff
|
||||
const maxRetries = 3
|
||||
const baseDelay = 1000 // 1 second
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
// Calculate entry slippage if expected price provided
|
||||
let entrySlippagePct: number | undefined
|
||||
if (params.expectedEntryPrice && params.entrySlippage !== undefined) {
|
||||
entrySlippagePct = params.entrySlippage
|
||||
}
|
||||
|
||||
const trade = await prisma.trade.create({
|
||||
data: {
|
||||
positionId: params.positionId,
|
||||
symbol: params.symbol,
|
||||
direction: params.direction,
|
||||
entryPrice: params.entryPrice,
|
||||
entryTime: new Date(),
|
||||
entrySlippage: params.entrySlippage,
|
||||
positionSizeUSD: params.positionSizeUSD, // NOTIONAL value (with leverage)
|
||||
collateralUSD: params.positionSizeUSD / params.leverage, // ACTUAL collateral used
|
||||
leverage: params.leverage,
|
||||
stopLossPrice: params.stopLossPrice,
|
||||
softStopPrice: params.softStopPrice,
|
||||
hardStopPrice: params.hardStopPrice,
|
||||
takeProfit1Price: params.takeProfit1Price,
|
||||
takeProfit2Price: params.takeProfit2Price,
|
||||
tp1SizePercent: params.tp1SizePercent,
|
||||
tp2SizePercent: params.tp2SizePercent,
|
||||
entryOrderTx: params.entryOrderTx,
|
||||
tp1OrderTx: params.tp1OrderTx,
|
||||
tp2OrderTx: params.tp2OrderTx,
|
||||
slOrderTx: params.slOrderTx,
|
||||
softStopOrderTx: params.softStopOrderTx,
|
||||
hardStopOrderTx: params.hardStopOrderTx,
|
||||
configSnapshot: params.configSnapshot,
|
||||
signalSource: params.signalSource,
|
||||
signalStrength: params.signalStrength,
|
||||
timeframe: params.timeframe,
|
||||
status: params.status || 'open',
|
||||
isTestTrade: params.isTestTrade || false,
|
||||
// Market context
|
||||
expectedEntryPrice: params.expectedEntryPrice,
|
||||
entrySlippagePct: entrySlippagePct,
|
||||
fundingRateAtEntry: params.fundingRateAtEntry,
|
||||
atrAtEntry: params.atrAtEntry,
|
||||
adxAtEntry: params.adxAtEntry,
|
||||
rsiAtEntry: params.rsiAtEntry,
|
||||
volumeAtEntry: params.volumeAtEntry,
|
||||
pricePositionAtEntry: params.pricePositionAtEntry,
|
||||
signalQualityScore: params.signalQualityScore,
|
||||
indicatorVersion: params.indicatorVersion,
|
||||
// Phantom trade fields
|
||||
isPhantom: params.isPhantom || false,
|
||||
expectedSizeUSD: params.expectedSizeUSD,
|
||||
actualSizeUSD: params.actualSizeUSD,
|
||||
phantomReason: params.phantomReason,
|
||||
},
|
||||
})
|
||||
|
||||
// CRITICAL: Verify record actually exists in database
|
||||
await new Promise(resolve => setTimeout(resolve, 100)) // Small delay for DB propagation
|
||||
const verifyTrade = await prisma.trade.findUnique({
|
||||
where: { positionId: params.positionId },
|
||||
select: { id: true, positionId: true, symbol: true }
|
||||
})
|
||||
|
||||
if (!verifyTrade) {
|
||||
const errorMsg = `Database save verification FAILED - record not found after create`
|
||||
logCriticalError(errorMsg, {
|
||||
attempt,
|
||||
positionId: params.positionId,
|
||||
symbol: params.symbol,
|
||||
transactionSignature: params.entryOrderTx
|
||||
})
|
||||
|
||||
if (attempt < maxRetries) {
|
||||
const delay = baseDelay * Math.pow(2, attempt - 1)
|
||||
console.log(`⏳ Verification failed, retrying in ${delay}ms (attempt ${attempt}/${maxRetries})...`)
|
||||
await new Promise(resolve => setTimeout(resolve, delay))
|
||||
continue // Retry
|
||||
}
|
||||
|
||||
throw new Error(errorMsg)
|
||||
}
|
||||
|
||||
console.log(`📊 Trade record created & VERIFIED: ${trade.id}`)
|
||||
logDatabaseOperation('createTrade', true, {
|
||||
table: 'Trade',
|
||||
recordId: trade.id,
|
||||
retryAttempt: attempt
|
||||
})
|
||||
|
||||
return trade
|
||||
} catch (error) {
|
||||
const errorMsg = `Failed to create trade record (attempt ${attempt}/${maxRetries})`
|
||||
console.error(`❌ ${errorMsg}:`, error)
|
||||
|
||||
logDatabaseOperation('createTrade', false, {
|
||||
table: 'Trade',
|
||||
recordId: params.positionId,
|
||||
error: error,
|
||||
retryAttempt: attempt
|
||||
})
|
||||
|
||||
if (attempt < maxRetries) {
|
||||
const delay = baseDelay * Math.pow(2, attempt - 1)
|
||||
console.log(`⏳ Retrying in ${delay}ms...`)
|
||||
await new Promise(resolve => setTimeout(resolve, delay))
|
||||
continue
|
||||
}
|
||||
|
||||
// Final attempt failed - log to persistent file
|
||||
logCriticalError('Database save failed after all retries', {
|
||||
positionId: params.positionId,
|
||||
symbol: params.symbol,
|
||||
direction: params.direction,
|
||||
entryPrice: params.entryPrice,
|
||||
entryTime: new Date(),
|
||||
entrySlippage: params.entrySlippage,
|
||||
positionSizeUSD: params.positionSizeUSD, // NOTIONAL value (with leverage)
|
||||
collateralUSD: params.positionSizeUSD / params.leverage, // ACTUAL collateral used
|
||||
leverage: params.leverage,
|
||||
stopLossPrice: params.stopLossPrice,
|
||||
softStopPrice: params.softStopPrice,
|
||||
hardStopPrice: params.hardStopPrice,
|
||||
takeProfit1Price: params.takeProfit1Price,
|
||||
takeProfit2Price: params.takeProfit2Price,
|
||||
tp1SizePercent: params.tp1SizePercent,
|
||||
tp2SizePercent: params.tp2SizePercent,
|
||||
entryOrderTx: params.entryOrderTx,
|
||||
tp1OrderTx: params.tp1OrderTx,
|
||||
tp2OrderTx: params.tp2OrderTx,
|
||||
slOrderTx: params.slOrderTx,
|
||||
softStopOrderTx: params.softStopOrderTx,
|
||||
hardStopOrderTx: params.hardStopOrderTx,
|
||||
configSnapshot: params.configSnapshot,
|
||||
signalSource: params.signalSource,
|
||||
signalStrength: params.signalStrength,
|
||||
timeframe: params.timeframe,
|
||||
status: params.status || 'open',
|
||||
isTestTrade: params.isTestTrade || false,
|
||||
// Market context
|
||||
expectedEntryPrice: params.expectedEntryPrice,
|
||||
entrySlippagePct: entrySlippagePct,
|
||||
fundingRateAtEntry: params.fundingRateAtEntry,
|
||||
atrAtEntry: params.atrAtEntry,
|
||||
adxAtEntry: params.adxAtEntry,
|
||||
rsiAtEntry: params.rsiAtEntry,
|
||||
volumeAtEntry: params.volumeAtEntry,
|
||||
pricePositionAtEntry: params.pricePositionAtEntry,
|
||||
signalQualityScore: params.signalQualityScore,
|
||||
indicatorVersion: params.indicatorVersion,
|
||||
// Phantom trade fields
|
||||
isPhantom: params.isPhantom || false,
|
||||
expectedSizeUSD: params.expectedSizeUSD,
|
||||
actualSizeUSD: params.actualSizeUSD,
|
||||
phantomReason: params.phantomReason,
|
||||
},
|
||||
})
|
||||
|
||||
console.log(`📊 Trade record created: ${trade.id}`)
|
||||
return trade
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to create trade record:', error)
|
||||
throw error
|
||||
transactionSignature: params.entryOrderTx,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
stack: error instanceof Error ? error.stack : undefined
|
||||
})
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Database save failed: max retries exceeded')
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user