feat: Automated multi-timeframe price tracking system

Implemented comprehensive price tracking for multi-timeframe signal analysis. **Components Added:** - lib/analysis/blocked-signal-tracker.ts - Background job tracking prices - app/api/analytics/signal-tracking/route.ts - Status/metrics endpoint **Features:** - Automatic price tracking at 1min, 5min, 15min, 30min intervals - TP1/TP2/SL hit detection using ATR-based targets - Max favorable/adverse excursion tracking (MFE/MAE) - Analysis completion after 30 minutes - Background job runs every 5 minutes - Entry price captured from signal time **Database Changes:** - Added entryPrice field to BlockedSignal (for price tracking baseline) - Added maxFavorablePrice, maxAdversePrice fields - Added maxFavorableExcursion, maxAdverseExcursion fields **Integration:** - Auto-starts on container startup - Tracks all DATA_COLLECTION_ONLY signals - Uses same TP/SL calculation as live trades (ATR-based) - Calculates profit % based on direction (long vs short) **API Endpoints:** - GET /api/analytics/signal-tracking - View tracking status and metrics - POST /api/analytics/signal-tracking - Manually trigger update (auth required) **Purpose:** Enables data-driven multi-timeframe comparison. After 50+ signals per timeframe, can analyze which timeframe (5min vs 15min vs 1H vs 4H vs Daily) has best win rate, profit potential, and signal quality. **What It Tracks:** - Price at 1min, 5min, 15min, 30min after signal - Would TP1/TP2/SL have been hit? - Maximum profit/loss during 30min window - Complete analysis of signal profitability **How It Works:** 1. Signal comes in (15min, 1H, 4H, Daily) → saved to BlockedSignal 2. Background job runs every 5min 3. Queries current price from Pyth 4. Calculates profit % from entry 5. Checks if TP/SL thresholds crossed 6. Updates MFE/MAE if new highs/lows 7. After 30min, marks analysisComplete=true **Future Analysis:** After 50+ signals per timeframe: - Compare TP1 hit rates across timeframes - Identify which timeframe has highest win rate - Determine optimal signal frequency vs quality trade-off - Switch production to best-performing timeframe User requested: "i want all the bells and whistles. lets make the powerhouse more powerfull. i cant see any reason why we shouldnt"
2025-11-19 17:18:47 +01:00
parent e1bce56065
commit 60fc571aa6
7 changed files with 487 additions and 3 deletions
--- a/app/api/analytics/signal-tracking/route.ts
+++ b/app/api/analytics/signal-tracking/route.ts
@@ -0,0 +1,166 @@
+/**
+ * Blocked Signal Tracking Status API
+ * 
+ * GET: View tracking status and statistics
+ * POST: Manually trigger tracking update (requires auth)
+ */
+
+import { NextRequest, NextResponse } from 'next/server'
+import { getPrismaClient } from '@/lib/database/trades'
+import { getBlockedSignalTracker } from '@/lib/analysis/blocked-signal-tracker'
+
+// GET: View tracking status
+export async function GET(request: NextRequest) {
+  try {
+    const prisma = getPrismaClient()
+    
+    // Get tracking statistics
+    const total = await prisma.blockedSignal.count({
+      where: { blockReason: 'DATA_COLLECTION_ONLY' }
+    })
+    
+    const incomplete = await prisma.blockedSignal.count({
+      where: {
+        blockReason: 'DATA_COLLECTION_ONLY',
+        analysisComplete: false
+      }
+    })
+    
+    const complete = await prisma.blockedSignal.count({
+      where: {
+        blockReason: 'DATA_COLLECTION_ONLY',
+        analysisComplete: true
+      }
+    })
+    
+    // Get completion rates by timeframe
+    const byTimeframe = await prisma.blockedSignal.groupBy({
+      by: ['timeframe'],
+      where: { blockReason: 'DATA_COLLECTION_ONLY' },
+      _count: { id: true }
+    })
+    
+    // Get signals with price data
+    const withPriceData = await prisma.blockedSignal.count({
+      where: {
+        blockReason: 'DATA_COLLECTION_ONLY',
+        priceAfter1Min: { not: null }
+      }
+    })
+    
+    // Get TP/SL hit rates
+    const tp1Hits = await prisma.blockedSignal.count({
+      where: {
+        blockReason: 'DATA_COLLECTION_ONLY',
+        wouldHitTP1: true
+      }
+    })
+    
+    const slHits = await prisma.blockedSignal.count({
+      where: {
+        blockReason: 'DATA_COLLECTION_ONLY',
+        wouldHitSL: true
+      }
+    })
+    
+    // Get recent tracked signals
+    const recentSignals = await prisma.blockedSignal.findMany({
+      where: { blockReason: 'DATA_COLLECTION_ONLY' },
+      select: {
+        id: true,
+        timeframe: true,
+        symbol: true,
+        direction: true,
+        signalQualityScore: true,
+        priceAfter1Min: true,
+        priceAfter5Min: true,
+        priceAfter15Min: true,
+        priceAfter30Min: true,
+        wouldHitTP1: true,
+        wouldHitTP2: true,
+        wouldHitSL: true,
+        analysisComplete: true,
+        createdAt: true
+      },
+      orderBy: { createdAt: 'desc' },
+      take: 10
+    })
+    
+    return NextResponse.json({
+      success: true,
+      tracking: {
+        total,
+        complete,
+        incomplete,
+        completionRate: total > 0 ? ((complete / total) * 100).toFixed(1) : '0.0'
+      },
+      byTimeframe: byTimeframe.map(tf => ({
+        timeframe: tf.timeframe,
+        count: tf._count.id
+      })),
+      metrics: {
+        withPriceData,
+        tp1Hits,
+        slHits,
+        tp1HitRate: complete > 0 ? ((tp1Hits / complete) * 100).toFixed(1) : '0.0',
+        slHitRate: complete > 0 ? ((slHits / complete) * 100).toFixed(1) : '0.0'
+      },
+      recentSignals: recentSignals.map((signal: any) => ({
+        id: signal.id,
+        timeframe: `${signal.timeframe}min`,
+        symbol: signal.symbol,
+        direction: signal.direction,
+        quality: signal.signalQualityScore,
+        price1min: signal.priceAfter1Min,
+        price5min: signal.priceAfter5Min,
+        price15min: signal.priceAfter15Min,
+        price30min: signal.priceAfter30Min,
+        hitTP1: signal.wouldHitTP1,
+        hitTP2: signal.wouldHitTP2,
+        hitSL: signal.wouldHitSL,
+        complete: signal.analysisComplete,
+        time: signal.createdAt
+      }))
+    })
+  } catch (error) {
+    console.error('Error getting signal tracking status:', error)
+    return NextResponse.json(
+      { success: false, error: 'Failed to get tracking status' },
+      { status: 500 }
+    )
+  }
+}
+
+// POST: Manually trigger tracking update
+export async function POST(request: NextRequest) {
+  try {
+    // Check auth
+    const authHeader = request.headers.get('Authorization')
+    const apiKey = process.env.API_SECRET_KEY
+    
+    if (!authHeader || !apiKey || authHeader !== `Bearer ${apiKey}`) {
+      return NextResponse.json(
+        { success: false, error: 'Unauthorized' },
+        { status: 401 }
+      )
+    }
+    
+    const tracker = getBlockedSignalTracker()
+    
+    // Trigger manual update by restarting
+    console.log('🔄 Manual tracking update triggered')
+    tracker.stop()
+    tracker.start()
+    
+    return NextResponse.json({
+      success: true,
+      message: 'Tracking update triggered'
+    })
+  } catch (error) {
+    console.error('Error triggering tracking update:', error)
+    return NextResponse.json(
+      { success: false, error: 'Failed to trigger update' },
+      { status: 500 }
+    )
+  }
+}
--- a/app/api/trading/check-risk/route.ts
+++ b/app/api/trading/check-risk/route.ts
@@ -147,6 +147,18 @@ export async function POST(request: NextRequest): Promise<NextResponse<RiskCheck
    
    console.log('🔍 Risk check for:', body)
    
+    // 🔬 MULTI-TIMEFRAME DATA COLLECTION
+    // Allow all non-5min signals to bypass risk checks (they'll be saved as data collection in execute endpoint)
+    const timeframe = body.timeframe || '5'
+    if (timeframe !== '5') {
+      console.log(`📊 DATA COLLECTION: ${timeframe}min signal bypassing risk checks (will save in execute endpoint)`)
+      return NextResponse.json({
+        allowed: true,
+        reason: 'Multi-timeframe data collection',
+        details: `${timeframe}min signal will be saved for analysis but not executed`,
+      })
+    }
+    
    const config = getMergedConfig()
    
    // Check for existing positions on the same symbol
--- a/app/api/trading/execute/route.ts
+++ b/app/api/trading/execute/route.ts
@@ -14,6 +14,7 @@ import { getInitializedPositionManager, ActiveTrade } from '@/lib/trading/positi
 import { createTrade, updateTradeExit } from '@/lib/database/trades'
 import { scoreSignalQuality } from '@/lib/trading/signal-quality'
 import { getMarketDataCache } from '@/lib/trading/market-data-cache'
+import { getPythPriceMonitor } from '@/lib/pyth/price-monitor'

 export interface ExecuteTradeRequest {
  symbol: string // TradingView symbol (e.g., 'SOLUSDT')
@@ -111,6 +112,11 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
    if (timeframe !== '5') {
      console.log(`📊 DATA COLLECTION: ${timeframe}min signal from ${driftSymbol}, saving for analysis (not executing)`)
      
+      // Get current price for entry tracking
+      const priceMonitor = getPythPriceMonitor()
+      const latestPrice = priceMonitor.getCachedPrice(driftSymbol)
+      const currentPrice = latestPrice?.price || body.signalPrice || 0
+      
      // Save to BlockedSignal for cross-timeframe analysis
      const { createBlockedSignal } = await import('@/lib/database/trades')
      try {
@@ -125,13 +131,13 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
          volumeRatio: body.volumeRatio,
          pricePosition: body.pricePosition,
          timeframe: timeframe,
-          signalPrice: body.signalPrice || 0,
+          signalPrice: currentPrice,
          signalQualityScore: 0, // Not scored since not executed
          signalQualityVersion: 'data-collection',
          minScoreRequired: 0,
          scoreBreakdown: {},
        })
-        console.log(`✅ ${timeframe}min signal saved to database for future analysis`)
+        console.log(`✅ ${timeframe}min signal saved at $${currentPrice.toFixed(2)} for future analysis`)
      } catch (dbError) {
        console.error(`❌ Failed to save ${timeframe}min signal:`, dbError)
      }
--- a/lib/analysis/blocked-signal-tracker.ts
+++ b/lib/analysis/blocked-signal-tracker.ts
@@ -0,0 +1,284 @@
+/**
+ * Blocked Signal Price Tracker
+ * 
+ * Automatically tracks price movements for blocked signals to determine
+ * if they would have been profitable trades. This enables data-driven
+ * multi-timeframe analysis.
+ * 
+ * Features:
+ * - Price tracking at 1min, 5min, 15min, 30min intervals
+ * - TP1/TP2/SL hit detection using ATR-based targets
+ * - Max favorable/adverse excursion tracking
+ * - Automatic analysis completion after 30 minutes
+ * - Background job runs every 5 minutes
+ */
+
+import { getPrismaClient } from '../database/trades'
+import { getPythPriceMonitor } from '../pyth/price-monitor'
+import { getMergedConfig } from '../../config/trading'
+
+interface BlockedSignalWithTracking {
+  id: string
+  symbol: string
+  direction: 'long' | 'short'
+  entryPrice: number
+  atr: number
+  adx: number
+  createdAt: Date
+  priceAfter1Min: number | null
+  priceAfter5Min: number | null
+  priceAfter15Min: number | null
+  priceAfter30Min: number | null
+  wouldHitTP1: boolean | null
+  wouldHitTP2: boolean | null
+  wouldHitSL: boolean | null
+  maxFavorablePrice: number | null
+  maxAdversePrice: number | null
+  maxFavorableExcursion: number | null
+  maxAdverseExcursion: number | null
+  analysisComplete: boolean
+}
+
+export class BlockedSignalTracker {
+  private prisma = getPrismaClient()
+  private intervalId: NodeJS.Timeout | null = null
+  private isRunning = false
+
+  /**
+   * Start the background tracking job
+   * Runs every 5 minutes to update price data for blocked signals
+   */
+  public start(): void {
+    if (this.isRunning) {
+      console.log('⚠️ Blocked signal tracker already running')
+      return
+    }
+
+    console.log('🔬 Starting blocked signal price tracker...')
+    this.isRunning = true
+
+    // Run immediately on start
+    this.trackPrices().catch(error => {
+      console.error('❌ Error in initial price tracking:', error)
+    })
+
+    // Then run every 5 minutes
+    this.intervalId = setInterval(() => {
+      this.trackPrices().catch(error => {
+        console.error('❌ Error in price tracking:', error)
+      })
+    }, 5 * 60 * 1000) // 5 minutes
+
+    console.log('✅ Blocked signal tracker started (runs every 5 minutes)')
+  }
+
+  /**
+   * Stop the background tracking job
+   */
+  public stop(): void {
+    if (this.intervalId) {
+      clearInterval(this.intervalId)
+      this.intervalId = null
+    }
+    this.isRunning = false
+    console.log('⏹️ Blocked signal tracker stopped')
+  }
+
+  /**
+   * Main tracking logic - processes all incomplete blocked signals
+   */
+  private async trackPrices(): Promise<void> {
+    try {
+      // Get all incomplete signals from last 24 hours
+      const signals = await this.prisma.blockedSignal.findMany({
+        where: {
+          blockReason: 'DATA_COLLECTION_ONLY',
+          analysisComplete: false,
+          createdAt: {
+            gte: new Date(Date.now() - 24 * 60 * 60 * 1000) // Last 24 hours
+          }
+        },
+        orderBy: { createdAt: 'asc' }
+      })
+
+      if (signals.length === 0) {
+        console.log('📊 No blocked signals to track')
+        return
+      }
+
+      console.log(`📊 Tracking ${signals.length} blocked signals...`)
+
+      for (const signal of signals) {
+        await this.trackSignal(signal as any)
+      }
+
+      console.log(`✅ Price tracking complete for ${signals.length} signals`)
+    } catch (error) {
+      console.error('❌ Error in trackPrices:', error)
+    }
+  }
+
+  /**
+   * Track a single blocked signal
+   */
+  private async trackSignal(signal: BlockedSignalWithTracking): Promise<void> {
+    try {
+      const now = Date.now()
+      const signalTime = signal.createdAt.getTime()
+      const elapsedMinutes = (now - signalTime) / (60 * 1000)
+
+      // Get current price
+      const priceMonitor = getPythPriceMonitor()
+      const latestPrice = priceMonitor.getCachedPrice(signal.symbol)
+      
+      if (!latestPrice || !latestPrice.price) {
+        console.log(`⚠️ No price available for ${signal.symbol}, skipping`)
+        return
+      }
+
+      const currentPrice = latestPrice.price
+      const entryPrice = Number(signal.entryPrice)
+
+      // Calculate profit percentage
+      const profitPercent = this.calculateProfitPercent(
+        entryPrice,
+        currentPrice,
+        signal.direction
+      )
+
+      // Calculate TP/SL levels using ATR
+      const config = getMergedConfig()
+      const { tp1Percent, tp2Percent, slPercent } = this.calculateTargets(
+        Number(signal.atr),
+        entryPrice,
+        config
+      )
+
+      // Update prices at intervals
+      const updates: any = {}
+
+      if (elapsedMinutes >= 1 && !signal.priceAfter1Min) {
+        updates.priceAfter1Min = currentPrice
+        console.log(`  📍 ${signal.symbol} ${signal.direction} @ 1min: $${currentPrice.toFixed(2)} (${profitPercent.toFixed(2)}%)`)
+      }
+
+      if (elapsedMinutes >= 5 && !signal.priceAfter5Min) {
+        updates.priceAfter5Min = currentPrice
+        console.log(`  📍 ${signal.symbol} ${signal.direction} @ 5min: $${currentPrice.toFixed(2)} (${profitPercent.toFixed(2)}%)`)
+      }
+
+      if (elapsedMinutes >= 15 && !signal.priceAfter15Min) {
+        updates.priceAfter15Min = currentPrice
+        console.log(`  📍 ${signal.symbol} ${signal.direction} @ 15min: $${currentPrice.toFixed(2)} (${profitPercent.toFixed(2)}%)`)
+      }
+
+      if (elapsedMinutes >= 30 && !signal.priceAfter30Min) {
+        updates.priceAfter30Min = currentPrice
+        updates.analysisComplete = true
+        console.log(`  ✅ ${signal.symbol} ${signal.direction} @ 30min: $${currentPrice.toFixed(2)} (${profitPercent.toFixed(2)}%) - COMPLETE`)
+      }
+
+      // Update max favorable/adverse excursion
+      const currentMFE = signal.maxFavorableExcursion || 0
+      const currentMAE = signal.maxAdverseExcursion || 0
+
+      if (profitPercent > currentMFE) {
+        updates.maxFavorableExcursion = profitPercent
+        updates.maxFavorablePrice = currentPrice
+      }
+
+      if (profitPercent < currentMAE) {
+        updates.maxAdverseExcursion = profitPercent
+        updates.maxAdversePrice = currentPrice
+      }
+
+      // Check if TP1/TP2/SL would have been hit
+      if (signal.wouldHitTP1 === null && Math.abs(profitPercent) >= tp1Percent) {
+        updates.wouldHitTP1 = profitPercent > 0
+        console.log(`  🎯 ${signal.symbol} ${signal.direction} hit ${profitPercent > 0 ? 'TP1' : 'SL'} (${profitPercent.toFixed(2)}%)`)
+      }
+
+      if (signal.wouldHitTP2 === null && Math.abs(profitPercent) >= tp2Percent) {
+        updates.wouldHitTP2 = profitPercent > 0
+        console.log(`  🎯 ${signal.symbol} ${signal.direction} hit TP2 (${profitPercent.toFixed(2)}%)`)
+      }
+
+      if (signal.wouldHitSL === null && profitPercent <= -slPercent) {
+        updates.wouldHitSL = true
+        console.log(`  🛑 ${signal.symbol} ${signal.direction} hit SL (${profitPercent.toFixed(2)}%)`)
+      }
+
+      // Update database if we have changes
+      if (Object.keys(updates).length > 0) {
+        await this.prisma.blockedSignal.update({
+          where: { id: signal.id },
+          data: updates
+        })
+      }
+    } catch (error) {
+      console.error(`❌ Error tracking signal ${signal.id}:`, error)
+    }
+  }
+
+  /**
+   * Calculate profit percentage based on direction
+   */
+  private calculateProfitPercent(
+    entryPrice: number,
+    currentPrice: number,
+    direction: 'long' | 'short'
+  ): number {
+    if (direction === 'long') {
+      return ((currentPrice - entryPrice) / entryPrice) * 100
+    } else {
+      return ((entryPrice - currentPrice) / entryPrice) * 100
+    }
+  }
+
+  /**
+   * Calculate TP/SL targets using ATR
+   */
+  private calculateTargets(
+    atr: number,
+    entryPrice: number,
+    config: any
+  ): { tp1Percent: number; tp2Percent: number; slPercent: number } {
+    // ATR as percentage of price
+    const atrPercent = (atr / entryPrice) * 100
+
+    // TP1: ATR × 2.0 multiplier
+    let tp1Percent = atrPercent * config.atrMultiplierTp1
+    tp1Percent = Math.max(config.minTp1Percent, Math.min(config.maxTp1Percent, tp1Percent))
+
+    // TP2: ATR × 4.0 multiplier
+    let tp2Percent = atrPercent * config.atrMultiplierTp2
+    tp2Percent = Math.max(config.minTp2Percent, Math.min(config.maxTp2Percent, tp2Percent))
+
+    // SL: ATR × 3.0 multiplier
+    let slPercent = atrPercent * config.atrMultiplierSl
+    slPercent = Math.max(config.minSlPercent, Math.min(config.maxSlPercent, slPercent))
+
+    return { tp1Percent, tp2Percent, slPercent }
+  }
+}
+
+// Singleton instance
+let trackerInstance: BlockedSignalTracker | null = null
+
+export function getBlockedSignalTracker(): BlockedSignalTracker {
+  if (!trackerInstance) {
+    trackerInstance = new BlockedSignalTracker()
+  }
+  return trackerInstance
+}
+
+export function startBlockedSignalTracking(): void {
+  const tracker = getBlockedSignalTracker()
+  tracker.start()
+}
+
+export function stopBlockedSignalTracking(): void {
+  if (trackerInstance) {
+    trackerInstance.stop()
+  }
+}
--- a/lib/database/trades.ts
+++ b/lib/database/trades.ts
@@ -504,6 +504,7 @@ export async function createBlockedSignal(params: CreateBlockedSignalParams) {
        direction: params.direction,
        timeframe: params.timeframe,
        signalPrice: params.signalPrice,
+        entryPrice: params.signalPrice, // Use signal price as entry for tracking
        atr: params.atr,
        adx: params.adx,
        rsi: params.rsi,
--- a/lib/startup/init-position-manager.ts
+++ b/lib/startup/init-position-manager.ts
@@ -9,6 +9,7 @@ import { getInitializedPositionManager } from '../trading/position-manager'
 import { initializeDriftService } from '../drift/client'
 import { getPrismaClient } from '../database/trades'
 import { getMarketConfig } from '../../config/trading'
+import { startBlockedSignalTracking } from '../analysis/blocked-signal-tracker'

 let initStarted = false

@@ -42,6 +43,10 @@ export async function initializePositionManagerOnStartup() {
    if (status.activeTradesCount > 0) {
      console.log(`📊 Monitoring: ${status.symbols.join(', ')}`)
    }
+    
+    // Start blocked signal price tracking
+    console.log('🔬 Starting blocked signal price tracker...')
+    startBlockedSignalTracking()
  } catch (error) {
    console.error('❌ Failed to initialize Position Manager on startup:', error)
  }
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -181,9 +181,12 @@ model BlockedSignal {
  indicatorVersion      String?  // Pine Script version (v5, v6, etc.)
  
  // Block reason
-  blockReason      String   // "QUALITY_SCORE_TOO_LOW", "DUPLICATE", "COOLDOWN", etc.
+  blockReason      String   // "QUALITY_SCORE_TOO_LOW", "DUPLICATE", "COOLDOWN", "DATA_COLLECTION_ONLY", etc.
  blockDetails     String?  // Human-readable details
  
+  // Entry tracking (for multi-timeframe analysis)
+  entryPrice       Float    @default(0) // Price at signal time
+  
  // For later analysis: track if it would have been profitable
  priceAfter1Min   Float?   // Price 1 minute after (filled by monitoring job)
  priceAfter5Min   Float?   // Price 5 minutes after
@@ -192,6 +195,13 @@ model BlockedSignal {
  wouldHitTP1      Boolean? // Would TP1 have been hit?
  wouldHitTP2      Boolean? // Would TP2 have been hit?
  wouldHitSL       Boolean? // Would SL have been hit?
+  
+  // Max favorable/adverse excursion (mirror Trade model)
+  maxFavorablePrice     Float?   // Price at max profit
+  maxAdversePrice       Float?   // Price at max loss
+  maxFavorableExcursion Float?   // Best profit % during tracking
+  maxAdverseExcursion   Float?   // Worst loss % during tracking
+  
  analysisComplete Boolean  @default(false) // Has post-analysis been done?
  
  @@index([symbol])