From 25776413d0e6c55c1a3a55e77f424f9af4bdd5a5 Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Fri, 14 Nov 2025 22:55:14 +0100 Subject: [PATCH] feat: Add signalSource field to identify manual vs TradingView trades - Set signalSource='manual' for Telegram trades, 'tradingview' for TradingView - Updated analytics queries to exclude manual trades from indicator analysis - getTradingStats() filters manual trades (TradingView performance only) - Version comparison endpoint filters manual trades - Created comprehensive filtering guide: docs/MANUAL_TRADE_FILTERING.md - Ensures clean data for indicator optimization without contamination --- app/api/analytics/version-comparison/route.ts | 2 + app/api/trading/execute/route.ts | 1 + docs/MANUAL_TRADE_FILTERING.md | 231 ++++++++++++++++++ lib/database/views.ts | 4 + 4 files changed, 238 insertions(+) create mode 100644 docs/MANUAL_TRADE_FILTERING.md diff --git a/app/api/analytics/version-comparison/route.ts b/app/api/analytics/version-comparison/route.ts index b90d1b6..8a05287 100644 --- a/app/api/analytics/version-comparison/route.ts +++ b/app/api/analytics/version-comparison/route.ts @@ -56,6 +56,7 @@ export async function GET() { WHERE "exitReason" IS NOT NULL AND "exitReason" NOT LIKE '%CLEANUP%' AND "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') GROUP BY "indicatorVersion" ORDER BY version DESC ` @@ -78,6 +79,7 @@ export async function GET() { WHERE "exitReason" IS NOT NULL AND "exitReason" NOT LIKE '%CLEANUP%' AND "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') AND "pricePositionAtEntry" < 15 GROUP BY "indicatorVersion" ORDER BY version DESC diff --git a/app/api/trading/execute/route.ts b/app/api/trading/execute/route.ts index bf29b78..56105ce 100644 --- a/app/api/trading/execute/route.ts +++ b/app/api/trading/execute/route.ts @@ -650,6 +650,7 @@ export async function POST(request: NextRequest): Promise 0 THEN 1 ELSE 0 END) as wins, + ROUND(100.0 * SUM(CASE WHEN "realizedPnL" > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) as win_rate, + ROUND(SUM("realizedPnL")::numeric, 2) as total_pnl, + ROUND(AVG("realizedPnL")::numeric, 2) as avg_pnl +FROM "Trade" +WHERE + "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') + AND "exitReason" IS NOT NULL + AND "exitReason" NOT LIKE '%CLEANUP%' +GROUP BY "indicatorVersion" +ORDER BY version DESC; +``` + +### Signal Quality Score Distribution (TradingView Only) + +```sql +SELECT + CASE + WHEN "signalQualityScore" >= 70 THEN '70-79' + WHEN "signalQualityScore" >= 65 THEN '65-69' + WHEN "signalQualityScore" >= 60 THEN '60-64' + ELSE '<60' + END as score_tier, + COUNT(*) as trades, + ROUND(100.0 * SUM(CASE WHEN "realizedPnL" > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) as win_rate, + ROUND(AVG("realizedPnL")::numeric, 2) as avg_pnl +FROM "Trade" +WHERE + "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') + AND "exitReason" IS NOT NULL + AND "signalQualityScore" IS NOT NULL +GROUP BY score_tier +ORDER BY MIN("signalQualityScore") DESC; +``` + +### Extreme Position Analysis (TradingView Only) + +```sql +-- Trades at range extremes (< 15% price position) +SELECT + "direction", + COUNT(*) as trades, + ROUND(100.0 * SUM(CASE WHEN "realizedPnL" > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) as win_rate, + ROUND(AVG("adxAtEntry")::numeric, 1) as avg_adx, + ROUND(AVG("pricePositionAtEntry")::numeric, 1) as avg_price_pos, + ROUND(SUM("realizedPnL")::numeric, 2) as total_pnl +FROM "Trade" +WHERE + "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') + AND "exitReason" IS NOT NULL + AND "pricePositionAtEntry" < 15 +GROUP BY "direction"; +``` + +### Timeframe Performance (TradingView Only) + +```sql +SELECT + "timeframe", + COUNT(*) as trades, + ROUND(100.0 * SUM(CASE WHEN "realizedPnL" > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) as win_rate, + ROUND(AVG("signalQualityScore")::numeric, 1) as avg_score, + ROUND(SUM("realizedPnL")::numeric, 2) as total_pnl +FROM "Trade" +WHERE + "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') + AND "exitReason" IS NOT NULL + AND "timeframe" IS NOT NULL +GROUP BY "timeframe" +ORDER BY timeframe; +``` + +## Queries INCLUDING Manual Trades + +Sometimes you DO want to see manual trades (e.g., overall account performance): + +```sql +-- ALL trades (manual + TradingView) +SELECT + "signalSource", + COUNT(*) as trades, + ROUND(SUM("realizedPnL")::numeric, 2) as total_pnl +FROM "Trade" +WHERE + "isTestTrade" = false + AND "exitReason" IS NOT NULL +GROUP BY "signalSource" +ORDER BY "signalSource"; +``` + +Output: +``` +signalSource | trades | total_pnl +-------------|--------|---------- +tradingview | 150 | +45.23 +manual | 11 | -12.56 +(null) | 10 | +8.90 (old trades before signalSource added) +``` + +## Application Code + +### TypeScript/Prisma Queries + +```typescript +// TradingView indicator analysis only +const trades = await prisma.trade.findMany({ + where: { + isTestTrade: false, + OR: [ + { signalSource: null }, // Old trades + { signalSource: { not: 'manual' } }, // Exclude Telegram + ], + exitReason: { not: null }, + NOT: { + exitReason: { contains: 'CLEANUP' }, + }, + }, +}) + +// All real trades (including manual) +const allTrades = await prisma.trade.findMany({ + where: { + isTestTrade: false, + exitReason: { not: null }, + }, +}) +``` + +## Migration for Old Trades + +Old trades don't have `signalSource` set (NULL). The filter `OR signalSource IS NULL` includes them in TradingView analysis, which is correct because they came from TradingView before the field existed. + +To backfill old trades: + +```sql +-- Set signalSource for old trades (if timeframe is known) +UPDATE "Trade" +SET "signalSource" = CASE + WHEN "timeframe" = 'manual' THEN 'manual' + WHEN "timeframe" IS NOT NULL AND "timeframe" != 'manual' THEN 'tradingview' + ELSE 'tradingview' -- Default assumption for old trades +END +WHERE "signalSource" IS NULL; +``` + +**Note:** Only run this if you want to clean up old data. Current filter handles NULL correctly. + +## Implementation Status + +**Updated files (Nov 14, 2025):** +- ✅ `app/api/trading/execute/route.ts` - Sets `signalSource` on new trades +- ✅ `app/api/analytics/version-comparison/route.ts` - Filters manual trades in SQL +- ✅ `lib/database/views.ts` - Filters manual trades in `getTradingStats()` + +**Future updates needed:** +- Analytics dashboard frontend (if it queries trades directly) +- Any custom SQL analysis scripts in `scripts/` directory +- Performance analysis queries in optimization roadmaps + +## Testing + +Verify filtering works: + +```sql +-- Should return 0 (no manual trades in TradingView analysis) +SELECT COUNT(*) +FROM "Trade" +WHERE + "isTestTrade" = false + AND ("signalSource" IS NULL OR "signalSource" != 'manual') + AND "exitReason" IS NOT NULL + AND "signalSource" = 'manual'; +``` + +Expected result: `0` (manual trades correctly excluded) + +## Summary + +**Rule of thumb:** +- **Indicator optimization:** Exclude manual trades (use filter above) +- **Account performance:** Include all real trades (exclude only `isTestTrade`) +- **Position sizing analysis:** Can include manual trades (they use same TP/SL system) + +**When in doubt:** Use the standard filter shown at the top of this document. + +--- + +*This filtering ensures clean data for TradingView indicator analysis while preserving manual trades for account-level reporting.* diff --git a/lib/database/views.ts b/lib/database/views.ts index af68184..b864fa5 100644 --- a/lib/database/views.ts +++ b/lib/database/views.ts @@ -143,6 +143,10 @@ export async function getTradingStats(days: number = 30) { createdAt: { gte: since }, status: 'closed', isTestTrade: false, // Real trades only + OR: [ + { signalSource: null }, // Old trades without signalSource + { signalSource: { not: 'manual' } }, // Exclude manual Telegram trades + ], }, })