feat: Add phantom trade detection and database tracking

- Detect position size mismatches (>50% variance) after opening - Save phantom trades to database with expectedSizeUSD, actualSizeUSD, phantomReason - Return error from execute endpoint to prevent Position Manager tracking - Add comprehensive documentation of phantom trade issue and solution - Enable data collection for pattern analysis and future optimization Fixes oracle price lag issue during volatile markets where transactions confirm but positions don't actually open at expected size.
2025-11-04 10:34:38 +01:00
parent f682b93a1e
commit 8bc08955cc
6 changed files with 313 additions and 3 deletions
--- a/app/api/trading/execute/route.ts
+++ b/app/api/trading/execute/route.ts
@@ -372,6 +372,68 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
        { status: 500 }
      )
    }
+    
+    // CRITICAL: Check for phantom trade (position opened but size mismatch)
+    if (openResult.isPhantom) {
+      console.error(`🚨 PHANTOM TRADE DETECTED - Not adding to Position Manager`)
+      console.error(`   Expected: $${positionSizeUSD.toFixed(2)}`)
+      console.error(`   Actual: $${openResult.actualSizeUSD?.toFixed(2)}`)
+      
+      // Save phantom trade to database for analysis
+      try {
+        const qualityScore = calculateQualityScore({
+          atr: body.atr,
+          adx: body.adx,
+          rsi: body.rsi,
+          volumeRatio: body.volumeRatio,
+          pricePosition: body.pricePosition,
+          direction: body.direction,
+        })
+        
+        await createTrade({
+          positionId: openResult.transactionSignature!,
+          symbol: driftSymbol,
+          direction: body.direction,
+          entryPrice: openResult.fillPrice!,
+          positionSizeUSD: positionSizeUSD,
+          leverage: config.leverage,
+          stopLossPrice: 0, // Not applicable for phantom
+          takeProfit1Price: 0,
+          takeProfit2Price: 0,
+          tp1SizePercent: 0,
+          tp2SizePercent: 0,
+          configSnapshot: config,
+          entryOrderTx: openResult.transactionSignature!,
+          signalStrength: body.signalStrength,
+          timeframe: body.timeframe,
+          atrAtEntry: body.atr,
+          adxAtEntry: body.adx,
+          rsiAtEntry: body.rsi,
+          volumeAtEntry: body.volumeRatio,
+          pricePositionAtEntry: body.pricePosition,
+          signalQualityScore: qualityScore,
+          // Phantom-specific fields
+          status: 'phantom',
+          isPhantom: true,
+          expectedSizeUSD: positionSizeUSD,
+          actualSizeUSD: openResult.actualSizeUSD,
+          phantomReason: 'ORACLE_PRICE_MISMATCH', // Likely cause based on logs
+        })
+        
+        console.log(`💾 Phantom trade saved to database for analysis`)
+      } catch (dbError) {
+        console.error('❌ Failed to save phantom trade:', dbError)
+      }
+      
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Phantom trade detected',
+          message: `Position opened but size mismatch detected. Expected $${positionSizeUSD.toFixed(2)}, got $${openResult.actualSizeUSD?.toFixed(2)}. This usually indicates oracle price was stale or order was rejected by exchange.`,
+        },
+        { status: 500 }
+      )
+    }

    // Calculate stop loss and take profit prices
    const entryPrice = openResult.fillPrice!
--- a/docs/history/PHANTOM_TRADE_DETECTION.md
+++ b/docs/history/PHANTOM_TRADE_DETECTION.md
@@ -0,0 +1,203 @@
+# Phantom Trade Detection & Prevention
+
+**Date:** November 4, 2025  
+**Issue:** SOL-PERP SHORT position showed as opened in Telegram but no actual position existed on Drift
+
+## Problem Description
+
+When a SHORT signal arrived after a LONG position:
+1. Bot closed LONG successfully (manual exit)
+2. Bot attempted to open SHORT for $2,100
+3. **Oracle price was $166.79 but actual market price was $158.51** (-5% discrepancy!)
+4. Drift rejected or partially filled the order (only 0.05 SOL = $8 instead of 12.59 SOL = $2,100)
+5. Position Manager detected size mismatch and marked as "phantom trade" with $0 P&L
+6. No actual SHORT position existed on Drift
+
+## Root Cause
+
+**Oracle price lag during volatile market movement:**
+- During signal flip, the market moved significantly
+- Oracle price hadn't updated to reflect actual market price
+- Drift rejected/partially filled order due to excessive price discrepancy
+- Transaction was confirmed on-chain but position was tiny/nonexistent
+
+## Solution Implemented
+
+### 1. **Enhanced Post-Entry Position Validation** ✅
+
+Modified `openPosition()` in `/lib/drift/orders.ts`:
+- After position opens, verify actual size vs expected size
+- Flag as "phantom" if actual size < 50% of expected
+- Return `isPhantom` flag and `actualSizeUSD` in result
+
+```typescript
+export interface OpenPositionResult {
+  success: boolean
+  transactionSignature?: string
+  fillPrice?: number
+  fillSize?: number
+  slippage?: number
+  error?: string
+  isPhantom?: boolean         // NEW: Position opened but size mismatch
+  actualSizeUSD?: number      // NEW: Actual position size from Drift
+}
+```
+
+### 2. **Phantom Trade Database Tracking** 📊
+
+Added new fields to `Trade` model in Prisma schema:
+```prisma
+status           String   @default("open") // "open", "closed", "failed", "phantom"
+isPhantom        Boolean  @default(false)
+expectedSizeUSD  Float?
+actualSizeUSD    Float?
+phantomReason    String?  // "ORACLE_PRICE_MISMATCH", "PARTIAL_FILL", "ORDER_REJECTED"
+```
+
+**Why track phantom trades:**
+- Measure how often this happens
+- Analyze conditions that cause phantoms (volatility, time of day, etc.)
+- Optimize entry logic based on data
+- Provide transparency in trade history
+
+### 3. **Immediate Phantom Detection in Execute Endpoint** 🚨
+
+Modified `/app/api/trading/execute/route.ts`:
+- After `openPosition()` returns, check `isPhantom` flag
+- If phantom detected:
+  - Save to database with `status: 'phantom'` and all metrics
+  - Log detailed error with expected vs actual size
+  - Return 500 error (prevents adding to Position Manager)
+  - NO cleanup needed (tiny position ignored, will auto-close eventually)
+
+```typescript
+if (openResult.isPhantom) {
+  console.error(`🚨 PHANTOM TRADE DETECTED - Not adding to Position Manager`)
+  
+  // Save for analysis
+  await createTrade({
+    ...params,
+    status: 'phantom',
+    isPhantom: true,
+    expectedSizeUSD: positionSizeUSD,
+    actualSizeUSD: openResult.actualSizeUSD,
+    phantomReason: 'ORACLE_PRICE_MISMATCH',
+  })
+  
+  return NextResponse.json({
+    success: false,
+    error: 'Phantom trade detected',
+    message: 'Oracle price mismatch - position not opened correctly'
+  }, { status: 500 })
+}
+```
+
+### 4. **What We Did NOT Implement** ❌
+
+Based on user preferences:
+
+- ❌ **20-minute cooldown:** Too long, defeats purpose of flips
+- ✅ **Keep 1-minute cooldown:** Already configured
+- ✅ **Use quality score:** Already implemented in check-risk endpoint
+- ❌ **Pre-entry oracle validation:** Not needed - post-entry detection is sufficient and catches the actual problem
+- ❌ **Auto-close phantom positions:** Not needed - tiny positions ignored
+
+## How It Works Now
+
+### Normal Trade Flow:
+1. Signal arrives → Check risk (quality score, cooldown, duplicates)
+2. Open position → Verify size matches expected
+3. If size OK → Place exit orders, add to Position Manager
+4. Monitor and exit normally
+
+### Phantom Trade Flow:
+1. Signal arrives → Check risk ✅
+2. Open position → Size mismatch detected! 🚨
+3. Save phantom trade to database 💾
+4. Return error, DO NOT add to Position Manager ❌
+5. Tiny position on Drift ignored (will expire/auto-close)
+
+## Database Analysis Queries
+
+```sql
+-- Count phantom trades
+SELECT COUNT(*) FROM "Trade" WHERE "isPhantom" = true;
+
+-- Phantom trades by symbol
+SELECT symbol, COUNT(*) as phantom_count, AVG("expectedSizeUSD") as avg_expected, AVG("actualSizeUSD") as avg_actual
+FROM "Trade" 
+WHERE "isPhantom" = true 
+GROUP BY symbol;
+
+-- Phantom trades by time of day (UTC)
+SELECT EXTRACT(HOUR FROM "createdAt") as hour, COUNT(*) as phantom_count
+FROM "Trade"
+WHERE "isPhantom" = true
+GROUP BY hour
+ORDER BY hour;
+
+-- Phantom trades with quality scores
+SELECT "signalQualityScore", COUNT(*) as count, AVG("atrAtEntry") as avg_atr
+FROM "Trade"
+WHERE "isPhantom" = true
+GROUP BY "signalQualityScore"
+ORDER BY "signalQualityScore" DESC;
+```
+
+## Expected Behavior
+
+### Telegram Notifications:
+- If phantom detected, execute endpoint returns 500 error
+- n8n workflow should catch this and send error notification
+- User sees: "Trade failed: Phantom trade detected"
+- NO "Position monitored" message
+
+### Dashboard:
+- Phantom trades appear in database with `status: 'phantom'`
+- Can be filtered out or analyzed separately
+- Shows expected vs actual size for debugging
+
+### Position Manager:
+- Phantom trades are NEVER added to Position Manager
+- No monitoring, no false alarms
+- No "closed externally" spam in logs
+
+## Prevention Strategy
+
+Going forward, phantom trades should be rare because:
+1. **1-minute cooldown** prevents rapid flips during volatility
+2. **Quality score filtering** blocks low-quality signals (which tend to occur during chaos)
+3. **Post-entry validation** catches phantoms immediately
+4. **Database tracking** allows us to analyze patterns and adjust
+
+If phantom trades continue to occur frequently, we can:
+- Increase cooldown for flips (2-3 minutes)
+- Add ATR-based volatility check (block flips when ATR > threshold)
+- Implement pre-entry oracle validation (add 2% discrepancy check before placing order)
+
+## Files Modified
+
+- `lib/drift/orders.ts` - Added phantom detection in `openPosition()`
+- `app/api/trading/execute/route.ts` - Added phantom handling after opening
+- `lib/database/trades.ts` - Added phantom fields to CreateTradeParams
+- `prisma/schema.prisma` - Added phantom trade fields to Trade model
+- `prisma/migrations/20251104091741_add_phantom_trade_fields/` - Database migration
+
+## Testing
+
+To test phantom detection:
+1. Modify `openPosition()` to simulate phantom (set actualSizeUSD = 10)
+2. Send test trade signal
+3. Verify:
+   - Error returned from execute endpoint
+   - Phantom trade saved to database with `isPhantom: true`
+   - NO position added to Position Manager
+   - Logs show "🚨 PHANTOM TRADE DETECTED"
+
+## Future Improvements
+
+If phantom trades remain an issue:
+1. **Auto-retry with delay:** Wait 5s for oracle to catch up, retry once
+2. **Oracle price validation:** Check Pyth price vs Drift oracle before placing order
+3. **Volatility-based cooldown:** Longer cooldown during high ATR periods
+4. **Symbol-specific thresholds:** SOL might need different validation than ETH
--- a/lib/database/trades.ts
+++ b/lib/database/trades.ts
@@ -52,6 +52,12 @@ export interface CreateTradeParams {
  volumeAtEntry?: number
  pricePositionAtEntry?: number
  signalQualityScore?: number
+  // Phantom trade fields
+  status?: string
+  isPhantom?: boolean
+  expectedSizeUSD?: number
+  actualSizeUSD?: number
+  phantomReason?: string
 }

 export interface UpdateTradeStateParams {
@@ -124,7 +130,7 @@ export async function createTrade(params: CreateTradeParams) {
        signalSource: params.signalSource,
        signalStrength: params.signalStrength,
        timeframe: params.timeframe,
-        status: 'open',
+        status: params.status || 'open',
        isTestTrade: params.isTestTrade || false,
        // Market context
        expectedEntryPrice: params.expectedEntryPrice,
@@ -136,6 +142,11 @@ export async function createTrade(params: CreateTradeParams) {
        volumeAtEntry: params.volumeAtEntry,
        pricePositionAtEntry: params.pricePositionAtEntry,
        signalQualityScore: params.signalQualityScore,
+        // Phantom trade fields
+        isPhantom: params.isPhantom || false,
+        expectedSizeUSD: params.expectedSizeUSD,
+        actualSizeUSD: params.actualSizeUSD,
+        phantomReason: params.phantomReason,
      },
    })
    
--- a/lib/drift/orders.ts
+++ b/lib/drift/orders.ts
@@ -29,6 +29,8 @@ export interface OpenPositionResult {
  fillSize?: number
  slippage?: number
  error?: string
+  isPhantom?: boolean // Position opened but size mismatch detected
+  actualSizeUSD?: number // Actual position size if different from requested
 }

 export interface ClosePositionParams {
@@ -179,16 +181,37 @@ export async function openPosition(
      const fillPrice = position.entryPrice
      const slippage = Math.abs((fillPrice - oraclePrice) / oraclePrice) * 100
      
+      // CRITICAL: Validate actual position size vs expected
+      // Phantom trade detection: Check if position is significantly smaller than expected
+      const actualSizeUSD = position.size * fillPrice
+      const expectedSizeUSD = params.sizeUSD
+      const sizeRatio = actualSizeUSD / expectedSizeUSD
+      
      console.log(`💰 Fill details:`)
      console.log(`   Fill price: $${fillPrice.toFixed(4)}`)
      console.log(`   Slippage: ${slippage.toFixed(3)}%`)
+      console.log(`   Expected size: $${expectedSizeUSD.toFixed(2)}`)
+      console.log(`   Actual size: $${actualSizeUSD.toFixed(2)}`)
+      console.log(`   Size ratio: ${(sizeRatio * 100).toFixed(1)}%`)
+      
+      // Flag as phantom if actual size is less than 50% of expected
+      const isPhantom = sizeRatio < 0.5
+      
+      if (isPhantom) {
+        console.error(`🚨 PHANTOM POSITION DETECTED!`)
+        console.error(`   Expected: $${expectedSizeUSD.toFixed(2)}`)
+        console.error(`   Actual: $${actualSizeUSD.toFixed(2)}`)
+        console.error(`   This indicates the order was rejected or partially filled by Drift`)
+      }
      
      return {
        success: true,
        transactionSignature: txSig,
        fillPrice,
-        fillSize: baseAssetSize,
+        fillSize: position.size, // Use actual size from Drift, not calculated
        slippage,
+        isPhantom,
+        actualSizeUSD,
      }
    } else {
      // Position not found yet (may be DRY_RUN mode)
--- a/prisma/migrations/20251104091741_add_phantom_trade_fields/migration.sql
+++ b/prisma/migrations/20251104091741_add_phantom_trade_fields/migration.sql
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "Trade" ADD COLUMN     "actualSizeUSD" DOUBLE PRECISION,
+ADD COLUMN     "expectedSizeUSD" DOUBLE PRECISION,
+ADD COLUMN     "isPhantom" BOOLEAN NOT NULL DEFAULT false,
+ADD COLUMN     "phantomReason" TEXT;
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -103,9 +103,15 @@ model Trade {
  timeframe        String?  // "5", "15", "60"
  
  // Status
-  status           String   @default("open") // "open", "closed", "failed"
+  status           String   @default("open") // "open", "closed", "failed", "phantom"
  isTestTrade      Boolean  @default(false) // Flag test trades for exclusion from analytics
  
+  // Phantom trade detection
+  isPhantom        Boolean  @default(false) // Position opened but size mismatch >50%
+  expectedSizeUSD  Float?   // Expected position size (when phantom)
+  actualSizeUSD    Float?   // Actual position size from Drift (when phantom)
+  phantomReason    String?  // "ORACLE_PRICE_MISMATCH", "PARTIAL_FILL", "ORDER_REJECTED"
+  
  // Relations
  priceUpdates     PriceUpdate[]