From 63b94016fe927383cea0b76e28b005e856684b8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Dec 2025 22:23:43 +0000 Subject: [PATCH] fix: Implement critical risk management fixes for bugs #76, #77, #78, #80 Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com> --- app/api/trading/execute/route.ts | 42 +++++ lib/drift/orders.ts | 241 ++++++++++++++----------- lib/monitoring/drift-state-verifier.ts | 46 ++++- lib/trading/position-manager.ts | 128 ++++++++++--- 4 files changed, 326 insertions(+), 131 deletions(-) diff --git a/app/api/trading/execute/route.ts b/app/api/trading/execute/route.ts index fd47e26..02a63cf 100644 --- a/app/api/trading/execute/route.ts +++ b/app/api/trading/execute/route.ts @@ -934,12 +934,54 @@ export async function POST(request: NextRequest): Promise= Math.floor(marketConfig.minOrderSize * 1e9)) { - const useDualStops = options.useDualStops ?? false if (useDualStops && options.softStopPrice && options.hardStopPrice) { // ============== DUAL STOP SYSTEM ============== logger.log('πŸ›‘οΈπŸ›‘οΈ Placing DUAL STOP SYSTEM...') - // 1. Soft Stop (TRIGGER_LIMIT) - Avoids wicks - const softStopBuffer = options.softStopBuffer ?? 0.4 - const softStopMultiplier = options.direction === 'long' - ? (1 - softStopBuffer / 100) - : (1 + softStopBuffer / 100) - - const softStopParams: any = { - orderType: OrderType.TRIGGER_LIMIT, - marketIndex: marketConfig.driftMarketIndex, - direction: orderDirection, - baseAssetAmount: new BN(slBaseAmount), - triggerPrice: new BN(Math.floor(options.softStopPrice * 1e6)), - price: new BN(Math.floor(options.softStopPrice * softStopMultiplier * 1e6)), - triggerCondition: options.direction === 'long' - ? OrderTriggerCondition.BELOW - : OrderTriggerCondition.ABOVE, - reduceOnly: true, + try { + // 1. Soft Stop (TRIGGER_LIMIT) - Avoids wicks + const softStopBuffer = options.softStopBuffer ?? 0.4 + const softStopMultiplier = options.direction === 'long' + ? (1 - softStopBuffer / 100) + : (1 + softStopBuffer / 100) + + const softStopParams: any = { + orderType: OrderType.TRIGGER_LIMIT, + marketIndex: marketConfig.driftMarketIndex, + direction: orderDirection, + baseAssetAmount: new BN(slBaseAmount), + triggerPrice: new BN(Math.floor(options.softStopPrice * 1e6)), + price: new BN(Math.floor(options.softStopPrice * softStopMultiplier * 1e6)), + triggerCondition: options.direction === 'long' + ? OrderTriggerCondition.BELOW + : OrderTriggerCondition.ABOVE, + reduceOnly: true, + } + + logger.log(` 1️⃣ Soft Stop (TRIGGER_LIMIT):`) + logger.log(` Trigger: $${options.softStopPrice.toFixed(4)}`) + logger.log(` Limit: $${(options.softStopPrice * softStopMultiplier).toFixed(4)}`) + logger.log(` Purpose: Avoid false breakouts/wicks`) + logger.log(` πŸ”„ Executing soft stop placement...`) + + const softStopSig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(softStopParams) + ) + logger.log(` βœ… Soft stop placed: ${softStopSig}`) + signatures.push(softStopSig) + } catch (softStopError) { + console.error(`❌ CRITICAL: Failed to place soft stop:`, softStopError) + throw new Error(`Soft stop placement failed: ${softStopError instanceof Error ? softStopError.message : 'Unknown error'}`) } - logger.log(` 1️⃣ Soft Stop (TRIGGER_LIMIT):`) - logger.log(` Trigger: $${options.softStopPrice.toFixed(4)}`) - logger.log(` Limit: $${(options.softStopPrice * softStopMultiplier).toFixed(4)}`) - logger.log(` Purpose: Avoid false breakouts/wicks`) - - const softStopSig = await retryWithBackoff(async () => - await (driftClient as any).placePerpOrder(softStopParams) - ) - logger.log(` βœ… Soft stop placed: ${softStopSig}`) - signatures.push(softStopSig) - - // 2. Hard Stop (TRIGGER_MARKET) - Guarantees exit - const hardStopParams: any = { - orderType: OrderType.TRIGGER_MARKET, - marketIndex: marketConfig.driftMarketIndex, - direction: orderDirection, - baseAssetAmount: new BN(slBaseAmount), - triggerPrice: new BN(Math.floor(options.hardStopPrice * 1e6)), - triggerCondition: options.direction === 'long' - ? OrderTriggerCondition.BELOW - : OrderTriggerCondition.ABOVE, - reduceOnly: true, + try { + // 2. Hard Stop (TRIGGER_MARKET) - Guarantees exit + const hardStopParams: any = { + orderType: OrderType.TRIGGER_MARKET, + marketIndex: marketConfig.driftMarketIndex, + direction: orderDirection, + baseAssetAmount: new BN(slBaseAmount), + triggerPrice: new BN(Math.floor(options.hardStopPrice * 1e6)), + triggerCondition: options.direction === 'long' + ? OrderTriggerCondition.BELOW + : OrderTriggerCondition.ABOVE, + reduceOnly: true, + } + + logger.log(` 2️⃣ Hard Stop (TRIGGER_MARKET):`) + logger.log(` Trigger: $${options.hardStopPrice.toFixed(4)}`) + logger.log(` Purpose: Guaranteed exit if soft stop doesn't fill`) + logger.log(` πŸ”„ Executing hard stop placement...`) + + const hardStopSig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(hardStopParams) + ) + logger.log(` βœ… Hard stop placed: ${hardStopSig}`) + signatures.push(hardStopSig) + } catch (hardStopError) { + console.error(`❌ CRITICAL: Failed to place hard stop:`, hardStopError) + throw new Error(`Hard stop placement failed: ${hardStopError instanceof Error ? hardStopError.message : 'Unknown error'}`) } - logger.log(` 2️⃣ Hard Stop (TRIGGER_MARKET):`) - logger.log(` Trigger: $${options.hardStopPrice.toFixed(4)}`) - logger.log(` Purpose: Guaranteed exit if soft stop doesn't fill`) - - const hardStopSig = await retryWithBackoff(async () => - await (driftClient as any).placePerpOrder(hardStopParams) - ) - logger.log(` βœ… Hard stop placed: ${hardStopSig}`) - signatures.push(hardStopSig) - logger.log(`🎯 Dual stop system active: Soft @ $${options.softStopPrice.toFixed(2)} | Hard @ $${options.hardStopPrice.toFixed(2)}`) } else { @@ -419,64 +436,86 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< const useStopLimit = options.useStopLimit ?? false const stopLimitBuffer = options.stopLimitBuffer ?? 0.5 - if (useStopLimit) { - // TRIGGER_LIMIT: For liquid markets - const limitPriceMultiplier = options.direction === 'long' - ? (1 - stopLimitBuffer / 100) - : (1 + stopLimitBuffer / 100) - - const orderParams: any = { - orderType: OrderType.TRIGGER_LIMIT, - marketIndex: marketConfig.driftMarketIndex, - direction: orderDirection, - baseAssetAmount: new BN(slBaseAmount), - triggerPrice: new BN(Math.floor(options.stopLossPrice * 1e6)), - price: new BN(Math.floor(options.stopLossPrice * limitPriceMultiplier * 1e6)), - triggerCondition: options.direction === 'long' - ? OrderTriggerCondition.BELOW - : OrderTriggerCondition.ABOVE, - reduceOnly: true, + try { + if (useStopLimit) { + // TRIGGER_LIMIT: For liquid markets + const limitPriceMultiplier = options.direction === 'long' + ? (1 - stopLimitBuffer / 100) + : (1 + stopLimitBuffer / 100) + + const orderParams: any = { + orderType: OrderType.TRIGGER_LIMIT, + marketIndex: marketConfig.driftMarketIndex, + direction: orderDirection, + baseAssetAmount: new BN(slBaseAmount), + triggerPrice: new BN(Math.floor(options.stopLossPrice * 1e6)), + price: new BN(Math.floor(options.stopLossPrice * limitPriceMultiplier * 1e6)), + triggerCondition: options.direction === 'long' + ? OrderTriggerCondition.BELOW + : OrderTriggerCondition.ABOVE, + reduceOnly: true, + } + + logger.log(`πŸ›‘οΈ Placing SL as TRIGGER_LIMIT (${stopLimitBuffer}% buffer)...`) + logger.log(` Trigger: ${options.direction === 'long' ? 'BELOW' : 'ABOVE'} $${options.stopLossPrice.toFixed(4)}`) + logger.log(` Limit: $${(options.stopLossPrice * limitPriceMultiplier).toFixed(4)}`) + logger.log(` ⚠️ May not fill during fast moves - use for liquid markets only!`) + logger.log(`πŸ”„ Executing SL trigger-limit placement...`) + + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) + logger.log('βœ… SL trigger-limit order placed:', sig) + signatures.push(sig) + } else { + // TRIGGER_MARKET: Default, guaranteed execution + const orderParams: any = { + orderType: OrderType.TRIGGER_MARKET, + marketIndex: marketConfig.driftMarketIndex, + direction: orderDirection, + baseAssetAmount: new BN(slBaseAmount), + triggerPrice: new BN(Math.floor(options.stopLossPrice * 1e6)), + triggerCondition: options.direction === 'long' + ? OrderTriggerCondition.BELOW + : OrderTriggerCondition.ABOVE, + reduceOnly: true, + } + + logger.log(`πŸ›‘οΈ Placing SL as TRIGGER_MARKET (guaranteed execution - RECOMMENDED)...`) + logger.log(` Trigger: ${options.direction === 'long' ? 'BELOW' : 'ABOVE'} $${options.stopLossPrice.toFixed(4)}`) + logger.log(` βœ… Will execute at market price when triggered (may slip but WILL fill)`) + logger.log(`πŸ”„ Executing SL trigger-market placement...`) + + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) + logger.log('βœ… SL trigger-market order placed:', sig) + signatures.push(sig) } - - logger.log(`πŸ›‘οΈ Placing SL as TRIGGER_LIMIT (${stopLimitBuffer}% buffer)...`) - logger.log(` Trigger: ${options.direction === 'long' ? 'BELOW' : 'ABOVE'} $${options.stopLossPrice.toFixed(4)}`) - logger.log(` Limit: $${(options.stopLossPrice * limitPriceMultiplier).toFixed(4)}`) - logger.log(` ⚠️ May not fill during fast moves - use for liquid markets only!`) - - const sig = await retryWithBackoff(async () => - await (driftClient as any).placePerpOrder(orderParams) - ) - logger.log('βœ… SL trigger-limit order placed:', sig) - signatures.push(sig) - } else { - // TRIGGER_MARKET: Default, guaranteed execution - const orderParams: any = { - orderType: OrderType.TRIGGER_MARKET, - marketIndex: marketConfig.driftMarketIndex, - direction: orderDirection, - baseAssetAmount: new BN(slBaseAmount), - triggerPrice: new BN(Math.floor(options.stopLossPrice * 1e6)), - triggerCondition: options.direction === 'long' - ? OrderTriggerCondition.BELOW - : OrderTriggerCondition.ABOVE, - reduceOnly: true, - } - - logger.log(`πŸ›‘οΈ Placing SL as TRIGGER_MARKET (guaranteed execution - RECOMMENDED)...`) - logger.log(` Trigger: ${options.direction === 'long' ? 'BELOW' : 'ABOVE'} $${options.stopLossPrice.toFixed(4)}`) - logger.log(` βœ… Will execute at market price when triggered (may slip but WILL fill)`) - - const sig = await retryWithBackoff(async () => - await (driftClient as any).placePerpOrder(orderParams) - ) - logger.log('βœ… SL trigger-market order placed:', sig) - signatures.push(sig) + } catch (slError) { + console.error(`❌ CRITICAL: Failed to place stop loss:`, slError) + throw new Error(`Stop loss placement failed: ${slError instanceof Error ? slError.message : 'Unknown error'}`) } } } else { logger.log('⚠️ SL size below market min, skipping on-chain SL') } + // CRITICAL VALIDATION (Bug #76 fix): Verify all expected orders were placed + if (signatures.length < expectedOrderCount) { + const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrderCount}, got ${signatures.length}. Position is UNPROTECTED!` + console.error(`❌ ${errorMsg}`) + console.error(` Expected: TP1 + TP2 + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`) + console.error(` Got ${signatures.length} signatures:`, signatures) + + return { + success: false, + error: errorMsg, + signatures // Return partial signatures for debugging + } + } + + logger.log(`βœ… All ${expectedOrderCount} exit orders placed successfully`) return { success: true, signatures } } catch (error) { console.error('❌ Failed to place exit orders:', error) diff --git a/lib/monitoring/drift-state-verifier.ts b/lib/monitoring/drift-state-verifier.ts index 89a8ee6..613f5dc 100644 --- a/lib/monitoring/drift-state-verifier.ts +++ b/lib/monitoring/drift-state-verifier.ts @@ -31,6 +31,9 @@ class DriftStateVerifier { private isRunning: boolean = false private checkIntervalMs: number = 10 * 60 * 1000 // 10 minutes private intervalId: NodeJS.Timeout | null = null + // BUG #80 FIX: Track close attempts per symbol to enforce cooldown + private recentCloseAttempts: Map = new Map() + private readonly COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes /** * Start the periodic verification service @@ -215,14 +218,30 @@ class DriftStateVerifier { /** * Retry closing a position that should be closed but isn't - * CRITICAL FIX (Dec 9, 2025): Stop retry loop if close transaction confirms + * BUG #80 FIX: Enhanced cooldown enforcement to prevent retry loops */ private async retryClose(mismatch: DriftStateMismatch): Promise { console.log(`πŸ”„ Retrying close for ${mismatch.symbol}...`) try { - // CRITICAL: Check if this trade already has a close attempt in progress - // If we recently tried to close (within 5 minutes), SKIP to avoid retry loop + // BUG #80 FIX: Check in-memory cooldown map first (faster than DB query) + const lastAttemptTime = this.recentCloseAttempts.get(mismatch.symbol) + + if (lastAttemptTime) { + const timeSinceAttempt = Date.now() - lastAttemptTime + + if (timeSinceAttempt < this.COOLDOWN_MS) { + const remainingCooldown = Math.ceil((this.COOLDOWN_MS - timeSinceAttempt) / 1000) + console.log(` ⏸️ COOLDOWN ACTIVE: Last attempt ${(timeSinceAttempt / 1000).toFixed(0)}s ago`) + console.log(` ⏳ Must wait ${remainingCooldown}s more before retry (5min cooldown)`) + console.log(` πŸ“Š Cooldown map state: ${Array.from(this.recentCloseAttempts.entries()).map(([s, t]) => `${s}:${Date.now()-t}ms`).join(', ')}`) + return + } else { + console.log(` βœ… Cooldown expired (${(timeSinceAttempt / 1000).toFixed(0)}s since last attempt)`) + } + } + + // ALSO check database for persistent cooldown tracking (survives restarts) const prisma = getPrismaClient() const trade = await prisma.trade.findUnique({ where: { id: mismatch.tradeId }, @@ -241,13 +260,23 @@ class DriftStateVerifier { const timeSinceRetry = Date.now() - lastRetryTime.getTime() // If we retried within last 5 minutes, SKIP (Drift propagation delay) - if (timeSinceRetry < 5 * 60 * 1000) { - console.log(` ⏳ Skipping retry - last attempt ${(timeSinceRetry / 1000).toFixed(0)}s ago (Drift propagation delay)`) + if (timeSinceRetry < this.COOLDOWN_MS) { + console.log(` ⏸️ DATABASE COOLDOWN: Last DB retry ${(timeSinceRetry / 1000).toFixed(0)}s ago`) + console.log(` ⏳ Drift propagation delay - skipping retry`) + + // Update in-memory map to match DB state + this.recentCloseAttempts.set(mismatch.symbol, lastRetryTime.getTime()) return } } } + console.log(` πŸš€ Proceeding with close attempt...`) + + // Record attempt time BEFORE calling closePosition + const attemptTime = Date.now() + this.recentCloseAttempts.set(mismatch.symbol, attemptTime) + const result = await closePosition({ symbol: mismatch.symbol, percentToClose: 100, @@ -268,15 +297,20 @@ class DriftStateVerifier { configSnapshot: { ...trade?.configSnapshot as any, retryCloseAttempted: true, - retryCloseTime: new Date().toISOString(), + retryCloseTime: new Date(attemptTime).toISOString(), } } }) + + console.log(` πŸ“ Cooldown recorded: ${mismatch.symbol} β†’ ${new Date(attemptTime).toISOString()}`) } else { console.error(` ❌ Failed to close ${mismatch.symbol}: ${result.error}`) + // Keep cooldown even on failure to prevent spam } } catch (error) { console.error(` ❌ Error retrying close for ${mismatch.symbol}:`, error) + // On error, still record attempt time to prevent rapid retries + this.recentCloseAttempts.set(mismatch.symbol, Date.now()) } } diff --git a/lib/trading/position-manager.ts b/lib/trading/position-manager.ts index 2dcb2f4..90c6719 100644 --- a/lib/trading/position-manager.ts +++ b/lib/trading/position-manager.ts @@ -267,31 +267,89 @@ export class PositionManager { // Start monitoring if not already running if (!this.isMonitoring && this.activeTrades.size > 0) { await this.startMonitoring() + + // BUG #77 FIX: Verify monitoring actually started + if (this.activeTrades.size > 0 && !this.isMonitoring) { + const errorMsg = `CRITICAL: Failed to start monitoring! activeTrades=${this.activeTrades.size}, isMonitoring=${this.isMonitoring}` + console.error(`❌ ${errorMsg}`) + + // Log to persistent file + const { logCriticalError } = await import('../utils/persistent-logger') + await logCriticalError('MONITORING_START_FAILED', { + activeTradesCount: this.activeTrades.size, + isMonitoring: this.isMonitoring, + symbols: Array.from(this.activeTrades.values()).map(t => t.symbol), + tradeIds: Array.from(this.activeTrades.keys()) + }) + + throw new Error(errorMsg) + } + + logger.log(`βœ… Monitoring verification passed: isMonitoring=${this.isMonitoring}`) } } /** * Remove a trade from monitoring + * BUG #78 FIX: Safely handle order cancellation to avoid removing active position orders */ async removeTrade(tradeId: string): Promise { const trade = this.activeTrades.get(tradeId) if (trade) { logger.log(`πŸ—‘οΈ Removing trade: ${trade.symbol}`) - // Cancel all orders for this symbol (cleanup orphaned orders) + // BUG #78 FIX: Check Drift position size before canceling orders + // If Drift shows an open position, DON'T cancel orders (may belong to active position) try { - const { cancelAllOrders } = await import('../drift/orders') - const cancelResult = await cancelAllOrders(trade.symbol) - if (cancelResult.success && cancelResult.cancelledCount! > 0) { - logger.log(`βœ… Cancelled ${cancelResult.cancelledCount} orphaned orders`) + const driftService = getDriftService() + const marketConfig = getMarketConfig(trade.symbol) + + // Query Drift for current position + const driftPosition = await driftService.getPosition(marketConfig.driftMarketIndex) + + if (driftPosition && Math.abs(driftPosition.size) >= 0.01) { + // Position still open on Drift - DO NOT cancel orders + console.warn(`⚠️ SAFETY CHECK: ${trade.symbol} position still open on Drift (size: ${driftPosition.size})`) + console.warn(` Skipping order cancellation to avoid removing active position protection`) + console.warn(` Removing from tracking only`) + + // Just remove from map, don't cancel orders + this.activeTrades.delete(tradeId) + + // Log for monitoring + const { logCriticalError } = await import('../utils/persistent-logger') + await logCriticalError('ORPHAN_REMOVAL_SKIPPED_ACTIVE_POSITION', { + tradeId, + symbol: trade.symbol, + driftSize: driftPosition.size, + reason: 'Drift position still open - preserved orders for safety' + }) + } else { + // Position confirmed closed on Drift - safe to cancel orders + logger.log(`βœ… Drift position confirmed closed (size: ${driftPosition?.size || 0})`) + logger.log(` Safe to cancel remaining orders`) + + const { cancelAllOrders } = await import('../drift/orders') + const cancelResult = await cancelAllOrders(trade.symbol) + + if (cancelResult.success && cancelResult.cancelledCount! > 0) { + logger.log(`βœ… Cancelled ${cancelResult.cancelledCount} orphaned orders`) + } else if (!cancelResult.success) { + console.error(`❌ Failed to cancel orders: ${cancelResult.error}`) + } else { + logger.log(`ℹ️ No orders to cancel`) + } + + this.activeTrades.delete(tradeId) } } catch (error) { - console.error('❌ Failed to cancel orders during trade removal:', error) - // Continue with removal even if cancel fails + console.error('❌ Error checking Drift position during trade removal:', error) + console.warn('⚠️ Removing from tracking without canceling orders (safety first)') + + // On error, err on side of caution - don't cancel orders + this.activeTrades.delete(tradeId) } - this.activeTrades.delete(tradeId) - // Stop monitoring if no more trades if (this.activeTrades.size === 0 && this.isMonitoring) { this.stopMonitoring() @@ -481,6 +539,7 @@ export class PositionManager { */ private async startMonitoring(): Promise { if (this.isMonitoring) { + logger.log('⚠️ Monitoring already active, skipping duplicate start') return } @@ -490,28 +549,49 @@ export class PositionManager { )] if (symbols.length === 0) { + logger.log('⚠️ No symbols to monitor, skipping start') return } - logger.log('πŸš€ Starting price monitoring for:', symbols) + logger.log('πŸš€ Starting price monitoring...') + logger.log(` Active trades: ${this.activeTrades.size}`) + logger.log(` Symbols: ${symbols.join(', ')}`) + logger.log(` Current isMonitoring: ${this.isMonitoring}`) const priceMonitor = getPythPriceMonitor() - await priceMonitor.start({ - symbols, - onPriceUpdate: async (update: PriceUpdate) => { - await this.handlePriceUpdate(update) - }, - onError: (error: Error) => { - console.error('❌ Price monitor error:', error) - }, - }) + try { + logger.log('πŸ“‘ Calling priceMonitor.start()...') + + await priceMonitor.start({ + symbols, + onPriceUpdate: async (update: PriceUpdate) => { + await this.handlePriceUpdate(update) + }, + onError: (error: Error) => { + console.error('❌ Price monitor error:', error) + }, + }) - this.isMonitoring = true - logger.log('βœ… Position monitoring active') - - // Schedule periodic validation to detect and cleanup ghost positions - this.scheduleValidation() + this.isMonitoring = true + logger.log('βœ… Position monitoring active') + logger.log(` isMonitoring flag set to: ${this.isMonitoring}`) + + // Schedule periodic validation to detect and cleanup ghost positions + this.scheduleValidation() + } catch (error) { + console.error('❌ CRITICAL: Failed to start price monitoring:', error) + + // Log error to persistent file + const { logCriticalError } = await import('../utils/persistent-logger') + await logCriticalError('PRICE_MONITOR_START_FAILED', { + symbols, + activeTradesCount: this.activeTrades.size, + error: error instanceof Error ? error.message : String(error) + }) + + throw error // Re-throw so caller knows monitoring failed + } } /**