Fix database persistence and add analytics

- Fixed Prisma client not being available in Docker container - Added isTestTrade flag to exclude test trades from analytics - Created analytics views for net positions (matches Drift UI netting) - Added API endpoints: /api/analytics/positions and /api/analytics/stats - Added test trade endpoint: /api/trading/test-db - Updated Dockerfile to properly copy Prisma client from builder stage - Database now successfully stores all trades with full details - Supports position netting calculations to match Drift perpetuals behavior
2025-10-27 09:35:01 +01:00
parent 1da5db5e75
commit 8e5c592cac
9 changed files with 589 additions and 8 deletions
--- a/.env
+++ b/.env
@@ -93,15 +93,15 @@ HARD_STOP_PERCENT=-2.5

 # Take Profit 1: Close 50% of position at this profit level
 # Example: +0.7% on 10x = +7% account gain
-TAKE_PROFIT_1_PERCENT=0.4
+TAKE_PROFIT_1_PERCENT=0.7

 # Take Profit 1 Size: What % of position to close at TP1
 # Example: 50 = close 50% of position
-TAKE_PROFIT_1_SIZE_PERCENT=75
+TAKE_PROFIT_1_SIZE_PERCENT=50

 # Take Profit 2: Close remaining 50% at this profit level
 # Example: +1.5% on 10x = +15% account gain
-TAKE_PROFIT_2_PERCENT=0.8
+TAKE_PROFIT_2_PERCENT=1.5

 # Take Profit 2 Size: What % of remaining position to close at TP2
 # Example: 100 = close all remaining position
@@ -113,7 +113,7 @@ EMERGENCY_STOP_PERCENT=-2

 # Dynamic stop-loss adjustments
 # Move SL to breakeven when profit reaches this level
-BREAKEVEN_TRIGGER_PERCENT=0.5
+BREAKEVEN_TRIGGER_PERCENT=0.7

 # Lock in profit when price reaches this level
 PROFIT_LOCK_TRIGGER_PERCENT=1.2
--- a/20
+++ b/20
@@ -27,10 +27,19 @@ RUN npm install --production && \
 # ================================
 FROM node:20-alpine AS builder

+# Install system dependencies for Prisma
+RUN apk add --no-cache \
+    python3 \
+    make \
+    g++ \
+    libc6-compat \
+    openssl
+
 WORKDIR /app

-# Copy dependencies from deps stage
-COPY --from=deps /app/node_modules ./node_modules
+# Copy package files and install ALL dependencies (including dev)
+COPY package*.json ./
+RUN npm install

 # Copy source code
 COPY . .
@@ -67,8 +76,11 @@ COPY --from=builder /app/package*.json ./
 COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
 COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static

-# Copy node_modules
-COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules
+# Copy Prisma schema and generated client from builder
+COPY --from=builder /app/prisma ./prisma
+
+# Copy node_modules from builder (includes Prisma client)
+COPY --from=builder --chown=nextjs:nodejs /app/node_modules ./node_modules

 # Set environment variables
 ENV NODE_ENV production
--- a/app/api/analytics/positions/route.ts
+++ b/app/api/analytics/positions/route.ts
@@ -0,0 +1,40 @@
+/**
+ * Position Analytics API
+ * 
+ * Shows net positions (what Drift displays) vs individual trades (what DB stores)
+ * GET /api/analytics/positions
+ */
+
+import { NextResponse } from 'next/server'
+import { getPositionSummary, getNetPositions, getTradesWithNetContext } from '@/lib/database/views'
+
+export async function GET() {
+  try {
+    const [summary, netPositions, tradesWithContext] = await Promise.all([
+      getPositionSummary(),
+      getNetPositions(),
+      getTradesWithNetContext(),
+    ])
+    
+    return NextResponse.json({
+      success: true,
+      summary,
+      netPositions,
+      individualTrades: tradesWithContext,
+      explanation: {
+        drift: 'Drift UI shows NET positions - opposite directions in same market cancel out',
+        database: 'Database stores individual trades to track entry/exit of each position',
+        example: 'If you have 0.3 SOL LONG and 0.5 SOL SHORT, Drift shows 0.2 SOL SHORT (net)',
+      },
+    })
+  } catch (error) {
+    console.error('❌ Error getting position analytics:', error)
+    return NextResponse.json(
+      {
+        success: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      },
+      { status: 500 }
+    )
+  }
+}
--- a/app/api/analytics/stats/route.ts
+++ b/app/api/analytics/stats/route.ts
@@ -0,0 +1,32 @@
+/**
+ * Trading Statistics API
+ * 
+ * Performance analytics excluding test trades
+ * GET /api/analytics/stats?days=30
+ */
+
+import { NextRequest, NextResponse } from 'next/server'
+import { getTradingStats } from '@/lib/database/views'
+
+export async function GET(request: NextRequest) {
+  try {
+    const searchParams = request.nextUrl.searchParams
+    const days = parseInt(searchParams.get('days') || '30')
+    
+    const stats = await getTradingStats(days)
+    
+    return NextResponse.json({
+      success: true,
+      stats,
+    })
+  } catch (error) {
+    console.error('❌ Error getting trading stats:', error)
+    return NextResponse.json(
+      {
+        success: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      },
+      { status: 500 }
+    )
+  }
+}
--- a/app/api/trading/test-db/route.ts
+++ b/app/api/trading/test-db/route.ts
@@ -0,0 +1,256 @@
+/**
+ * Test Database Trade Endpoint
+ * 
+ * Creates small test trades to verify database functionality
+ * POST /api/trading/test-db
+ */
+
+import { NextRequest, NextResponse } from 'next/server'
+import { initializeDriftService } from '@/lib/drift/client'
+import { openPosition, placeExitOrders } from '@/lib/drift/orders'
+import { getMergedConfig } from '@/config/trading'
+import { getPositionManager, ActiveTrade } from '@/lib/trading/position-manager'
+import { createTrade } from '@/lib/database/trades'
+
+export interface TestTradeRequest {
+  symbol?: string // Default: 'SOL-PERP'
+  direction?: 'long' | 'short' // Default: 'long'
+  sizeUSD?: number // Default: $10
+  leverage?: number // Default: 1x
+}
+
+export interface TestTradeResponse {
+  success: boolean
+  message?: string
+  trade?: {
+    id: string
+    positionId: string
+    symbol: string
+    direction: 'long' | 'short'
+    entryPrice: number
+    positionSize: number
+    leverage: number
+  }
+  error?: string
+}
+
+export async function POST(request: NextRequest): Promise<NextResponse<TestTradeResponse>> {
+  try {
+    console.log('🧪 Test trade request received')
+    
+    // Parse request body
+    const body: TestTradeRequest = await request.json().catch(() => ({}))
+    
+    // Use minimal settings for test trade
+    const symbol = body.symbol || 'SOL-PERP'
+    const direction = body.direction || 'long'
+    const baseSize = body.sizeUSD || 10 // $10 base collateral
+    const leverage = body.leverage || 1 // 1x leverage (safe)
+    
+    // IMPORTANT: For Drift, we pass the BASE size (collateral), not notional
+    // Drift internally applies the leverage
+    const positionSizeUSD = baseSize // This is the actual collateral/margin used
+    const notionalSize = baseSize * leverage // This is what shows in Drift UI
+    
+    console.log(`🧪 Creating TEST trade:`)
+    console.log(`   Symbol: ${symbol}`)
+    console.log(`   Direction: ${direction}`)
+    console.log(`   Collateral: $${baseSize}`)
+    console.log(`   Leverage: ${leverage}x`)
+    console.log(`   Notional size: $${notionalSize} (what you'll see in Drift)`)
+    console.log(`   ⚠️  Marked as TEST TRADE`)
+
+    // Get base config but override with test settings
+    const config = getMergedConfig({
+      positionSize: baseSize,
+      leverage: leverage,
+      stopLossPercent: -1.5,
+      takeProfit1Percent: 0.7,
+      takeProfit2Percent: 1.5,
+    })
+    
+    // Initialize Drift service
+    const driftService = await initializeDriftService()
+    
+    // Check account health
+    const health = await driftService.getAccountHealth()
+    console.log('💊 Account health:', health)
+    
+    if (health.freeCollateral <= 0) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Insufficient collateral',
+          message: `Free collateral: $${health.freeCollateral.toFixed(2)}`,
+        },
+        { status: 400 }
+      )
+    }
+
+    // Open position
+    const openResult = await openPosition({
+      symbol,
+      direction,
+      sizeUSD: positionSizeUSD,
+      slippageTolerance: config.slippageTolerance,
+    })
+
+    if (!openResult.success) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Position open failed',
+          message: openResult.error,
+        },
+        { status: 500 }
+      )
+    }
+
+    const entryPrice = openResult.fillPrice!
+    
+    // Calculate exit prices
+    const calculatePrice = (entry: number, percent: number, dir: 'long' | 'short') => {
+      if (dir === 'long') {
+        return entry * (1 + percent / 100)
+      } else {
+        return entry * (1 - percent / 100)
+      }
+    }
+    
+    const stopLossPrice = calculatePrice(entryPrice, config.stopLossPercent, direction)
+    const tp1Price = calculatePrice(entryPrice, config.takeProfit1Percent, direction)
+    const tp2Price = calculatePrice(entryPrice, config.takeProfit2Percent, direction)
+    const emergencyStopPrice = calculatePrice(entryPrice, config.emergencyStopPercent, direction)
+
+    console.log('📊 Test trade targets:')
+    console.log(`   Entry: $${entryPrice.toFixed(4)}`)
+    console.log(`   SL: $${stopLossPrice.toFixed(4)} (${config.stopLossPercent}%)`)
+    console.log(`   TP1: $${tp1Price.toFixed(4)} (${config.takeProfit1Percent}%)`)
+    console.log(`   TP2: $${tp2Price.toFixed(4)} (${config.takeProfit2Percent}%)`)
+
+    // Place exit orders
+    let exitOrderSignatures: string[] = []
+    try {
+      const exitRes = await placeExitOrders({
+        symbol,
+        positionSizeUSD,
+        tp1Price,
+        tp2Price,
+        stopLossPrice,
+        tp1SizePercent: config.takeProfit1SizePercent || 50,
+        tp2SizePercent: config.takeProfit2SizePercent || 100,
+        direction,
+        useDualStops: config.useDualStops,
+        softStopPrice: config.useDualStops ? calculatePrice(entryPrice, config.softStopPercent, direction) : undefined,
+        softStopBuffer: config.softStopBuffer,
+        hardStopPrice: config.useDualStops ? calculatePrice(entryPrice, config.hardStopPercent, direction) : undefined,
+      })
+
+      if (exitRes.success && exitRes.signatures) {
+        exitOrderSignatures = exitRes.signatures
+        console.log('📨 Exit orders placed:', exitRes.signatures)
+      }
+    } catch (err) {
+      console.error('❌ Failed to place exit orders:', err)
+    }
+
+    // Create active trade object for position manager
+    const activeTrade: ActiveTrade = {
+      id: `test-trade-${Date.now()}`,
+      positionId: openResult.transactionSignature!,
+      symbol,
+      direction,
+      entryPrice,
+      entryTime: Date.now(),
+      positionSize: positionSizeUSD,
+      leverage,
+      stopLossPrice,
+      tp1Price,
+      tp2Price,
+      emergencyStopPrice,
+      currentSize: positionSizeUSD,
+      tp1Hit: false,
+      slMovedToBreakeven: false,
+      slMovedToProfit: false,
+      realizedPnL: 0,
+      unrealizedPnL: 0,
+      peakPnL: 0,
+      priceCheckCount: 0,
+      lastPrice: entryPrice,
+      lastUpdateTime: Date.now(),
+    }
+
+    // Add to position manager
+    const positionManager = getPositionManager()
+    await positionManager.addTrade(activeTrade)
+    console.log('✅ Test trade added to position manager')
+
+    // Save to database with TEST flag
+    try {
+      const dbTrade = await createTrade({
+        positionId: openResult.transactionSignature!,
+        symbol,
+        direction,
+        entryPrice,
+        entrySlippage: openResult.slippage,
+        positionSizeUSD,
+        leverage,
+        stopLossPrice,
+        takeProfit1Price: tp1Price,
+        takeProfit2Price: tp2Price,
+        tp1SizePercent: config.takeProfit1SizePercent || 50,
+        tp2SizePercent: config.takeProfit2SizePercent || 100,
+        configSnapshot: config,
+        entryOrderTx: openResult.transactionSignature!,
+        tp1OrderTx: exitOrderSignatures[0],
+        tp2OrderTx: exitOrderSignatures[1],
+        slOrderTx: config.useDualStops ? undefined : exitOrderSignatures[2],
+        softStopOrderTx: config.useDualStops ? exitOrderSignatures[2] : undefined,
+        hardStopOrderTx: config.useDualStops ? exitOrderSignatures[3] : undefined,
+        signalSource: 'test-api',
+        signalStrength: 'test',
+        timeframe: 'test',
+        isTestTrade: true, // Mark as test trade
+      })
+      
+      console.log('💾✅ Test trade saved to database with ID:', dbTrade.id)
+      console.log('🏷️  Trade marked as TEST - will be excluded from analytics')
+
+      return NextResponse.json({
+        success: true,
+        message: `✅ Test trade created! Collateral: $${baseSize} | Leverage: ${leverage}x | Notional: $${notionalSize} ${direction} on ${symbol}`,
+        trade: {
+          id: dbTrade.id,
+          positionId: openResult.transactionSignature!,
+          symbol,
+          direction,
+          entryPrice,
+          positionSize: positionSizeUSD,
+          notionalSize: notionalSize,
+          leverage,
+        },
+      })
+      
+    } catch (dbError) {
+      console.error('❌ Failed to save test trade to database:', dbError)
+      
+      return NextResponse.json({
+        success: false,
+        error: 'Database save failed',
+        message: dbError instanceof Error ? dbError.message : 'Unknown database error',
+      }, { status: 500 })
+    }
+
+  } catch (error) {
+    console.error('❌ Test trade execution error:', error)
+    
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Internal server error',
+        message: error instanceof Error ? error.message : 'Unknown error',
+      },
+      { status: 500 }
+    )
+  }
+}
--- a/lib/database/trades.ts
+++ b/lib/database/trades.ts
@@ -42,6 +42,7 @@ export interface CreateTradeParams {
  signalSource?: string
  signalStrength?: string
  timeframe?: string
+  isTestTrade?: boolean
 }

 export interface UpdateTradeExitParams {
@@ -90,6 +91,7 @@ export async function createTrade(params: CreateTradeParams) {
        signalStrength: params.signalStrength,
        timeframe: params.timeframe,
        status: 'open',
+        isTestTrade: params.isTestTrade || false,
      },
    })
    
@@ -204,6 +206,7 @@ export async function getTradeStats(days: number = 30) {
    where: {
      createdAt: { gte: since },
      status: 'closed',
+      isTestTrade: false, // Exclude test trades from stats
    },
  })
  
--- a/lib/database/views.ts
+++ b/lib/database/views.ts
@@ -0,0 +1,235 @@
+/**
+ * Database Views and Analytics Queries
+ * 
+ * Provides net position calculations and analytics that match what you see on Drift
+ */
+
+import { getPrismaClient } from './trades'
+
+/**
+ * Get net positions across all open trades (matches Drift UI)
+ * 
+ * NOTE: Drift perpetuals NET opposite positions in the same market.
+ * If you have both LONG and SHORT positions in SOL-PERP, Drift shows only the net exposure.
+ */
+export async function getNetPositions() {
+  const prisma = getPrismaClient()
+  
+  const openTrades = await prisma.trade.findMany({
+    where: {
+      status: 'open',
+      isTestTrade: false, // Exclude test trades
+    },
+    select: {
+      symbol: true,
+      direction: true,
+      positionSizeUSD: true,
+      entryPrice: true,
+      leverage: true,
+    },
+  })
+  
+  // Group by symbol and calculate net positions
+  const netBySymbol = new Map<string, {
+    symbol: string
+    longUSD: number
+    shortUSD: number
+    longSOL: number
+    shortSOL: number
+    netUSD: number
+    netSOL: number
+    netDirection: 'long' | 'short' | 'flat'
+    avgLongEntry: number
+    avgShortEntry: number
+    tradeCount: number
+  }>()
+  
+  for (const trade of openTrades) {
+    const existing = netBySymbol.get(trade.symbol) || {
+      symbol: trade.symbol,
+      longUSD: 0,
+      shortUSD: 0,
+      longSOL: 0,
+      shortSOL: 0,
+      netUSD: 0,
+      netSOL: 0,
+      netDirection: 'flat' as const,
+      avgLongEntry: 0,
+      avgShortEntry: 0,
+      tradeCount: 0,
+    }
+    
+    const solSize = trade.positionSizeUSD / trade.entryPrice
+    
+    if (trade.direction === 'long') {
+      existing.longUSD += trade.positionSizeUSD
+      existing.longSOL += solSize
+      existing.avgLongEntry = existing.longUSD / existing.longSOL
+    } else {
+      existing.shortUSD += trade.positionSizeUSD
+      existing.shortSOL += solSize
+      existing.avgShortEntry = existing.shortUSD / existing.shortSOL
+    }
+    
+    existing.tradeCount++
+    netBySymbol.set(trade.symbol, existing)
+  }
+  
+  // Calculate net exposure
+  const results = []
+  for (const [symbol, data] of netBySymbol) {
+    data.netSOL = data.longSOL - data.shortSOL
+    data.netUSD = data.longUSD - data.shortUSD
+    
+    if (Math.abs(data.netSOL) < 0.001) {
+      data.netDirection = 'flat'
+    } else {
+      data.netDirection = data.netSOL > 0 ? 'long' : 'short'
+    }
+    
+    results.push(data)
+  }
+  
+  return results
+}
+
+/**
+ * Get individual trades with their contribution to net position
+ */
+export async function getTradesWithNetContext() {
+  const prisma = getPrismaClient()
+  
+  const trades = await prisma.trade.findMany({
+    where: {
+      status: 'open',
+      isTestTrade: false,
+    },
+    orderBy: {
+      createdAt: 'desc',
+    },
+  })
+  
+  const netPositions = await getNetPositions()
+  const netMap = new Map(netPositions.map(n => [n.symbol, n]))
+  
+  return trades.map(trade => {
+    const net = netMap.get(trade.symbol)
+    const solSize = trade.positionSizeUSD / trade.entryPrice
+    
+    return {
+      ...trade,
+      solSize,
+      netPosition: net ? {
+        netSOL: net.netSOL,
+        netUSD: net.netUSD,
+        netDirection: net.netDirection,
+        contributionPercent: Math.abs((solSize / net.netSOL) * 100),
+      } : null,
+    }
+  })
+}
+
+/**
+ * Get trading statistics (excludes test trades)
+ */
+export async function getTradingStats(days: number = 30) {
+  const prisma = getPrismaClient()
+  
+  const since = new Date()
+  since.setDate(since.getDate() - days)
+  
+  const trades = await prisma.trade.findMany({
+    where: {
+      createdAt: { gte: since },
+      status: 'closed',
+      isTestTrade: false, // Real trades only
+    },
+  })
+  
+  const testTrades = await prisma.trade.count({
+    where: {
+      createdAt: { gte: since },
+      isTestTrade: true,
+    },
+  })
+  
+  const winning = trades.filter((t) => (t.realizedPnL ?? 0) > 0)
+  const losing = trades.filter((t) => (t.realizedPnL ?? 0) < 0)
+  
+  const totalPnL = trades.reduce((sum, t) => sum + (t.realizedPnL ?? 0), 0)
+  const winRate = trades.length > 0 ? (winning.length / trades.length) * 100 : 0
+  
+  const avgWin = winning.length > 0
+    ? winning.reduce((sum, t) => sum + (t.realizedPnL ?? 0), 0) / winning.length
+    : 0
+    
+  const avgLoss = losing.length > 0
+    ? losing.reduce((sum, t) => sum + (t.realizedPnL ?? 0), 0) / losing.length
+    : 0
+  
+  const profitFactor = avgLoss !== 0 ? avgWin / Math.abs(avgLoss) : 0
+  
+  return {
+    period: `Last ${days} days`,
+    realTrades: {
+      total: trades.length,
+      winning: winning.length,
+      losing: losing.length,
+      winRate: `${winRate.toFixed(1)}%`,
+      totalPnL: `$${totalPnL.toFixed(2)}`,
+      avgWin: `$${avgWin.toFixed(2)}`,
+      avgLoss: `$${avgLoss.toFixed(2)}`,
+      profitFactor: profitFactor.toFixed(2),
+    },
+    testTrades: {
+      count: testTrades,
+      note: 'Excluded from statistics above',
+    },
+  }
+}
+
+/**
+ * Get position summary (what you see on Drift vs what's in database)
+ */
+export async function getPositionSummary() {
+  const prisma = getPrismaClient()
+  
+  const openTrades = await prisma.trade.count({
+    where: { status: 'open', isTestTrade: false },
+  })
+  
+  const openTestTrades = await prisma.trade.count({
+    where: { status: 'open', isTestTrade: true },
+  })
+  
+  const individualPositions = await prisma.trade.findMany({
+    where: { status: 'open', isTestTrade: false },
+    select: {
+      symbol: true,
+      direction: true,
+      positionSizeUSD: true,
+      entryPrice: true,
+    },
+  })
+  
+  const totalIndividualUSD = individualPositions.reduce(
+    (sum, t) => sum + t.positionSizeUSD, 0
+  )
+  
+  const netPositions = await getNetPositions()
+  const totalNetUSD = netPositions.reduce(
+    (sum, n) => sum + Math.abs(n.netUSD), 0
+  )
+  
+  return {
+    summary: {
+      individualTrades: openTrades,
+      testTrades: openTestTrades,
+      totalIndividualExposure: `$${totalIndividualUSD.toFixed(2)}`,
+      netExposure: `$${totalNetUSD.toFixed(2)}`,
+      explanation: 'Drift shows NET exposure (opposite positions cancel out)',
+    },
+    netPositions,
+    individualPositions,
+  }
+}
--- a/prisma/migrations/20251027080947_add_test_trade_flag/migration.sql
+++ b/prisma/migrations/20251027080947_add_test_trade_flag/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Trade" ADD COLUMN     "isTestTrade" BOOLEAN NOT NULL DEFAULT false;
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -68,6 +68,7 @@ model Trade {
  
  // Status
  status           String   @default("open") // "open", "closed", "failed"
+  isTestTrade      Boolean  @default(false) // Flag test trades for exclusion from analytics
  
  // Relations
  priceUpdates     PriceUpdate[]