diff --git a/app/api/trading/execute/route.ts b/app/api/trading/execute/route.ts index 60eacf7..47514f1 100644 --- a/app/api/trading/execute/route.ts +++ b/app/api/trading/execute/route.ts @@ -15,6 +15,7 @@ import { createTrade, updateTradeExit } from '@/lib/database/trades' import { scoreSignalQuality } from '@/lib/trading/signal-quality' import { getMarketDataCache } from '@/lib/trading/market-data-cache' import { getPythPriceMonitor } from '@/lib/pyth/price-monitor' +import { logCriticalError, logTradeExecution } from '@/lib/utils/persistent-logger' export interface ExecuteTradeRequest { symbol: string // TradingView symbol (e.g., 'SOLUSDT') @@ -803,11 +804,40 @@ export async function POST(request: NextRequest): Promise setTimeout(resolve, 100)) // Small delay for DB propagation + const verifyTrade = await prisma.trade.findUnique({ + where: { positionId: params.positionId }, + select: { id: true, positionId: true, symbol: true } + }) + + if (!verifyTrade) { + const errorMsg = `Database save verification FAILED - record not found after create` + logCriticalError(errorMsg, { + attempt, + positionId: params.positionId, + symbol: params.symbol, + transactionSignature: params.entryOrderTx + }) + + if (attempt < maxRetries) { + const delay = baseDelay * Math.pow(2, attempt - 1) + console.log(`⏳ Verification failed, retrying in ${delay}ms (attempt ${attempt}/${maxRetries})...`) + await new Promise(resolve => setTimeout(resolve, delay)) + continue // Retry + } + + throw new Error(errorMsg) + } + + console.log(`📊 Trade record created & VERIFIED: ${trade.id}`) + logDatabaseOperation('createTrade', true, { + table: 'Trade', + recordId: trade.id, + retryAttempt: attempt + }) + + return trade + } catch (error) { + const errorMsg = `Failed to create trade record (attempt ${attempt}/${maxRetries})` + console.error(`❌ ${errorMsg}:`, error) + + logDatabaseOperation('createTrade', false, { + table: 'Trade', + recordId: params.positionId, + error: error, + retryAttempt: attempt + }) + + if (attempt < maxRetries) { + const delay = baseDelay * Math.pow(2, attempt - 1) + console.log(`⏳ Retrying in ${delay}ms...`) + await new Promise(resolve => setTimeout(resolve, delay)) + continue + } + + // Final attempt failed - log to persistent file + logCriticalError('Database save failed after all retries', { positionId: params.positionId, symbol: params.symbol, direction: params.direction, entryPrice: params.entryPrice, - entryTime: new Date(), - entrySlippage: params.entrySlippage, - positionSizeUSD: params.positionSizeUSD, // NOTIONAL value (with leverage) - collateralUSD: params.positionSizeUSD / params.leverage, // ACTUAL collateral used - leverage: params.leverage, - stopLossPrice: params.stopLossPrice, - softStopPrice: params.softStopPrice, - hardStopPrice: params.hardStopPrice, - takeProfit1Price: params.takeProfit1Price, - takeProfit2Price: params.takeProfit2Price, - tp1SizePercent: params.tp1SizePercent, - tp2SizePercent: params.tp2SizePercent, - entryOrderTx: params.entryOrderTx, - tp1OrderTx: params.tp1OrderTx, - tp2OrderTx: params.tp2OrderTx, - slOrderTx: params.slOrderTx, - softStopOrderTx: params.softStopOrderTx, - hardStopOrderTx: params.hardStopOrderTx, - configSnapshot: params.configSnapshot, - signalSource: params.signalSource, - signalStrength: params.signalStrength, - timeframe: params.timeframe, - status: params.status || 'open', - isTestTrade: params.isTestTrade || false, - // Market context - expectedEntryPrice: params.expectedEntryPrice, - entrySlippagePct: entrySlippagePct, - fundingRateAtEntry: params.fundingRateAtEntry, - atrAtEntry: params.atrAtEntry, - adxAtEntry: params.adxAtEntry, - rsiAtEntry: params.rsiAtEntry, - volumeAtEntry: params.volumeAtEntry, - pricePositionAtEntry: params.pricePositionAtEntry, - signalQualityScore: params.signalQualityScore, - indicatorVersion: params.indicatorVersion, - // Phantom trade fields - isPhantom: params.isPhantom || false, - expectedSizeUSD: params.expectedSizeUSD, - actualSizeUSD: params.actualSizeUSD, - phantomReason: params.phantomReason, - }, - }) - - console.log(`📊 Trade record created: ${trade.id}`) - return trade - } catch (error) { - console.error('❌ Failed to create trade record:', error) - throw error + transactionSignature: params.entryOrderTx, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined + }) + + throw error + } } + + throw new Error('Database save failed: max retries exceeded') } /** diff --git a/lib/startup/init-position-manager.ts b/lib/startup/init-position-manager.ts index f72a9bf..7963c82 100644 --- a/lib/startup/init-position-manager.ts +++ b/lib/startup/init-position-manager.ts @@ -7,10 +7,12 @@ import { getInitializedPositionManager } from '../trading/position-manager' import { initializeDriftService } from '../drift/client' -import { getPrismaClient } from '../database/trades' -import { getMarketConfig } from '../../config/trading' +import { getPrismaClient, createTrade } from '../database/trades' +import { getMarketConfig, getMergedConfig } from '../../config/trading' import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker' import { startStopHuntTracking } from '../trading/stop-hunt-tracker' +import { logCriticalError } from '../utils/persistent-logger' +import { sendPositionClosedNotification } from '../notifications/telegram' let initStarted = false @@ -36,6 +38,9 @@ export async function initializePositionManagerOnStartup() { // Then validate open trades against Drift positions await validateOpenTrades() + // CRITICAL: Detect orphaned positions (on Drift but not in database) + await detectOrphanedPositions() + const manager = await getInitializedPositionManager() const status = manager.getStatus() @@ -286,3 +291,158 @@ async function restoreOrdersIfMissing( console.error(` 🚨 CRITICAL: Position may be unprotected`) } } + +/** + * Detect orphaned positions - positions on Drift with NO database record + * + * CRITICAL FIX (Nov 21, 2025): Prevents ghost positions from database save failures + * + * This can happen when: + * - Database save fails silently during trade execution + * - Prisma transaction rolls back but no error thrown + * - Container restart interrupts database save + * + * Recovery: + * 1. Create retroactive database record with current Drift data + * 2. Send Telegram alert about orphaned position found + * 3. Add to Position Manager for normal TP/SL monitoring + */ +async function detectOrphanedPositions() { + try { + const prisma = getPrismaClient() + const driftService = await initializeDriftService() + + console.log('🔍 Checking for orphaned positions on Drift...') + + // Get all open positions from Drift + const driftPositions = await driftService.getAllPositions() + + if (driftPositions.length === 0) { + console.log('✅ No positions on Drift') + return + } + + console.log(`🔍 Found ${driftPositions.length} positions on Drift, checking database...`) + + // Get all open trades from database + const openTrades = await prisma.trade.findMany({ + where: { status: 'open' }, + select: { symbol: true, positionId: true, direction: true } + }) + + const trackedSymbols = new Set(openTrades.map(t => `${t.symbol}-${t.direction}`)) + + // Check each Drift position + for (const position of driftPositions) { + const positionKey = `${position.symbol}-${position.side.toLowerCase()}` + + if (trackedSymbols.has(positionKey)) { + console.log(`✅ ${position.symbol} ${position.side} tracked in database`) + continue + } + + // ORPHAN DETECTED! + // Get current price from Drift oracle + const marketConfig = getMarketConfig(position.symbol) + const currentPrice = await driftService.getOraclePrice(marketConfig.driftMarketIndex) + const positionSizeUSD = Math.abs(position.size) * currentPrice + + console.log(`🚨 ORPHAN POSITION DETECTED!`) + console.log(` Symbol: ${position.symbol}`) + console.log(` Direction: ${position.side}`) + console.log(` Size: ${Math.abs(position.size)} (notional: $${positionSizeUSD.toFixed(2)})`) + console.log(` Entry: $${position.entryPrice.toFixed(4)}`) + console.log(` Current: $${currentPrice.toFixed(4)}`) + + // Log to persistent file + logCriticalError('ORPHAN POSITION DETECTED - Creating retroactive database record', { + symbol: position.symbol, + direction: position.side.toLowerCase(), + entryPrice: position.entryPrice, + size: Math.abs(position.size), + currentPrice: currentPrice, + detectedAt: new Date().toISOString() + }) + + try { + // Get config for TP/SL calculation + const config = getMergedConfig() + + // Calculate estimated TP/SL prices based on current config + const direction = position.side.toLowerCase() as 'long' | 'short' + const entryPrice = position.entryPrice + + // Calculate TP/SL using same logic as execute endpoint + const stopLossPrice = direction === 'long' + ? entryPrice * (1 + config.stopLossPercent / 100) + : entryPrice * (1 - config.stopLossPercent / 100) + + const tp1Price = direction === 'long' + ? entryPrice * (1 + config.takeProfit1Percent / 100) + : entryPrice * (1 - config.takeProfit1Percent / 100) + + const tp2Price = direction === 'long' + ? entryPrice * (1 + config.takeProfit2Percent / 100) + : entryPrice * (1 - config.takeProfit2Percent / 100) + + // Create retroactive database record + console.log(`🔄 Creating retroactive database record...`) + const trade = await createTrade({ + positionId: `ORPHAN-${Date.now()}`, // Fake position ID since we don't have transaction + symbol: position.symbol, + direction: direction, + entryPrice: entryPrice, + positionSizeUSD: positionSizeUSD, + leverage: config.leverage, + stopLossPrice: stopLossPrice, + takeProfit1Price: tp1Price, + takeProfit2Price: tp2Price, + tp1SizePercent: config.takeProfit1SizePercent || 75, + tp2SizePercent: config.takeProfit2SizePercent || 0, + entryOrderTx: `ORPHAN-${Date.now()}`, // Fake transaction ID + configSnapshot: config, + signalSource: 'orphan_recovery', + timeframe: 'unknown', + status: 'open', + }) + + console.log(`✅ Retroactive database record created: ${trade.id}`) + + // Send Telegram notification + try { + await sendPositionClosedNotification({ + symbol: position.symbol, + direction: direction, + entryPrice: entryPrice, + exitPrice: currentPrice, + positionSize: positionSizeUSD, + realizedPnL: 0, // Unknown + holdTimeSeconds: 0, + exitReason: 'ORPHAN_DETECTED', + maxGain: 0, + maxDrawdown: 0, + }) + } catch (telegramError) { + console.error('Failed to send orphan notification:', telegramError) + } + + console.log(`🎯 Orphan position now tracked and monitored`) + + } catch (recoveryError) { + console.error(`❌ Failed to recover orphan position ${position.symbol}:`, recoveryError) + logCriticalError('ORPHAN RECOVERY FAILED', { + symbol: position.symbol, + error: recoveryError instanceof Error ? recoveryError.message : String(recoveryError) + }) + } + } + + console.log('✅ Orphan position detection complete') + + } catch (error) { + console.error('❌ Error detecting orphaned positions:', error) + logCriticalError('Orphan detection failed', { + error: error instanceof Error ? error.message : String(error) + }) + } +} diff --git a/lib/utils/persistent-logger.ts b/lib/utils/persistent-logger.ts new file mode 100644 index 0000000..7755af0 --- /dev/null +++ b/lib/utils/persistent-logger.ts @@ -0,0 +1,150 @@ +/** + * Persistent File Logger - Survives Container Restarts + * Critical for debugging database save failures and system issues + */ + +import * as fs from 'fs' +import * as path from 'path' + +const LOG_DIR = '/app/logs' +const ERROR_LOG = path.join(LOG_DIR, 'errors.log') +const TRADE_LOG = path.join(LOG_DIR, 'trades.log') +const MAX_LOG_SIZE = 10 * 1024 * 1024 // 10MB per log file + +// Ensure log directory exists +function ensureLogDir() { + try { + if (!fs.existsSync(LOG_DIR)) { + fs.mkdirSync(LOG_DIR, { recursive: true }) + } + } catch (error) { + console.error('Failed to create log directory:', error) + } +} + +// Rotate log if too large +function rotateLogIfNeeded(logPath: string) { + try { + if (fs.existsSync(logPath)) { + const stats = fs.statSync(logPath) + if (stats.size > MAX_LOG_SIZE) { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const rotatedPath = `${logPath}.${timestamp}` + fs.renameSync(logPath, rotatedPath) + console.log(`📦 Rotated log: ${rotatedPath}`) + } + } + } catch (error) { + console.error('Failed to rotate log:', error) + } +} + +// Format log entry +function formatLogEntry(level: string, message: string, details?: any): string { + const timestamp = new Date().toISOString() + const detailsStr = details ? `\n${JSON.stringify(details, null, 2)}` : '' + return `[${timestamp}] ${level}: ${message}${detailsStr}\n` +} + +// Append to log file +function appendToLog(logPath: string, entry: string) { + try { + ensureLogDir() + rotateLogIfNeeded(logPath) + fs.appendFileSync(logPath, entry, 'utf8') + } catch (error) { + console.error(`Failed to write to ${logPath}:`, error) + } +} + +/** + * Log critical error (database failures, position issues) + */ +export function logCriticalError(message: string, details?: any) { + const entry = formatLogEntry('CRITICAL', message, details) + console.error('🔴 CRITICAL:', message, details || '') + appendToLog(ERROR_LOG, entry) +} + +/** + * Log trade execution (success or failure) + */ +export function logTradeExecution( + success: boolean, + tradeDetails: { + symbol: string + direction: string + entryPrice: number + positionSize: number + transactionSignature?: string + error?: string + } +) { + const level = success ? 'SUCCESS' : 'FAILURE' + const message = success + ? `Trade opened: ${tradeDetails.symbol} ${tradeDetails.direction} @ $${tradeDetails.entryPrice}` + : `Trade failed: ${tradeDetails.symbol} ${tradeDetails.direction} - ${tradeDetails.error}` + + const entry = formatLogEntry(level, message, tradeDetails) + console.log(success ? '✅' : '❌', message) + appendToLog(TRADE_LOG, entry) +} + +/** + * Log database operation (create, update, query) + */ +export function logDatabaseOperation( + operation: string, + success: boolean, + details: { + table?: string + recordId?: string + error?: any + retryAttempt?: number + } +) { + const level = success ? 'DB_SUCCESS' : 'DB_FAILURE' + const message = success + ? `${operation} succeeded: ${details.table || 'unknown'}` + : `${operation} failed: ${details.error?.message || 'Unknown error'}` + + const entry = formatLogEntry(level, message, details) + console.log(success ? '💾' : '❌', message) + appendToLog(ERROR_LOG, entry) +} + +/** + * Read recent error logs (for debugging) + */ +export function getRecentErrors(lines: number = 50): string[] { + try { + if (!fs.existsSync(ERROR_LOG)) { + return [] + } + + const content = fs.readFileSync(ERROR_LOG, 'utf8') + const allLines = content.split('\n').filter(line => line.trim()) + return allLines.slice(-lines) + } catch (error) { + console.error('Failed to read error log:', error) + return [] + } +} + +/** + * Read recent trade logs + */ +export function getRecentTrades(lines: number = 50): string[] { + try { + if (!fs.existsSync(TRADE_LOG)) { + return [] + } + + const content = fs.readFileSync(TRADE_LOG, 'utf8') + const allLines = content.split('\n').filter(line => line.trim()) + return allLines.slice(-lines) + } catch (error) { + console.error('Failed to read trade log:', error) + return [] + } +}