From 6590f4fb1e471446d5bf871c7d1153dab0d1ffad Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Fri, 14 Nov 2025 05:37:51 +0100 Subject: [PATCH] feat: phantom trade auto-closure system - Auto-close phantom positions immediately via market order - Return HTTP 200 (not 500) to allow n8n workflow continuation - Save phantom trades to database with full P&L tracking - Exit reason: 'manual' category for phantom auto-closes - Protects user during unavailable hours (sleeping, no phone) - Add Docker build best practices to instructions (background + tail) - Document phantom system as Critical Component #1 - Add Common Pitfall #30: Phantom notification workflow Why auto-close: - User can't always respond to phantom alerts - Unmonitored position = unlimited risk exposure - Better to exit with small loss/gain than leave exposed - Re-entry possible if setup actually good Files changed: - app/api/trading/execute/route.ts: Auto-close logic - .github/copilot-instructions.md: Documentation + build pattern --- .env | 4 +- .github/copilot-instructions.md | 155 +++++++++++++++++++++- app/api/trading/execute/route.ts | 103 ++++++++++++-- config/trading.ts | 2 +- docker-compose.yml | 11 ++ docker-compose.yml.backup-20251113-002225 | 121 +++++++++++++++++ lib/drift/client.ts | 4 + 7 files changed, 383 insertions(+), 17 deletions(-) create mode 100644 docker-compose.yml.backup-20251113-002225 diff --git a/.env b/.env index 56c2648..4ef70da 100644 --- a/.env +++ b/.env @@ -369,8 +369,8 @@ TRAILING_STOP_PERCENT=0.3 TRAILING_STOP_ACTIVATION=0.4 MIN_QUALITY_SCORE=60 SOLANA_ENABLED=true -SOLANA_POSITION_SIZE=100 -SOLANA_LEVERAGE=15 +SOLANA_POSITION_SIZE=50 +SOLANA_LEVERAGE=1 SOLANA_USE_PERCENTAGE_SIZE=true ETHEREUM_ENABLED=false ETHEREUM_POSITION_SIZE=50 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b11f806..67a2221 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -66,6 +66,8 @@ ## VERIFICATION MANDATE: Financial Code Requires Proof +**CRITICAL: THIS IS A REAL MONEY TRADING SYSTEM - NOT A TOY PROJECT** + **Core Principle:** In trading systems, "working" means "verified with real data", NOT "code looks correct". **NEVER declare something working without:** @@ -73,6 +75,14 @@ 2. Verifying database state matches expectations 3. Comparing calculated values to source data 4. Testing with real trades when applicable +5. **CONFIRMING CODE IS DEPLOYED** - Check container start time vs commit time + +**CODE COMMITTED ≠ CODE DEPLOYED** +- Git commit at 15:56 means NOTHING if container started at 15:06 +- ALWAYS verify: `docker logs trading-bot-v4 | grep "Server starting" | head -1` +- Compare container start time to commit timestamp +- If container older than commit: **CODE NOT DEPLOYED, FIX NOT ACTIVE** +- Never say "fixed" or "protected" until deployment verified ### Critical Path Verification Requirements @@ -206,6 +216,12 @@ Then observe logs on actual trade: ### Deployment Checklist +**MANDATORY PRE-DEPLOYMENT VERIFICATION:** +- [ ] Check container start time: `docker logs trading-bot-v4 | grep "Server starting" | head -1` +- [ ] Compare to commit timestamp: Container MUST be newer than code changes +- [ ] If container older: **STOP - Code not deployed, fix not active** +- [ ] Never declare "fixed" or "working" until container restarted with new code + Before marking feature complete: - [ ] Code review completed - [ ] Unit tests pass (if applicable) @@ -213,6 +229,7 @@ Before marking feature complete: - [ ] Logs show expected behavior - [ ] Database state verified with SQL - [ ] Edge cases tested +- [ ] **Container restarted and verified running new code** - [ ] Documentation updated (including Common Pitfalls if applicable) - [ ] User notified of what to verify during first real trade @@ -224,17 +241,113 @@ Before marking feature complete: - Test trade behaved differently than expected - You're unsure about unit conversions or SDK behavior - Change affects money (position sizing, P&L, exits) +- **Container hasn't been restarted since code commit** **Instead say:** - "Code is updated. Need to verify with test trade - watch for [specific log message]" - "Fixed, but requires verification: check database shows [expected value]" - "Deployed. First real trade should show [behavior]. If not, there's still a bug." +- **"Code committed but NOT deployed - container running old version, fix not active yet"** + +### Docker Build Best Practices + +**CRITICAL: Prevent build interruptions with background execution + live monitoring** + +Docker builds take 40-70 seconds and are easily interrupted by terminal issues. Use this pattern: + +```bash +# Start build in background with live log tail +cd /home/icke/traderv4 && docker compose build trading-bot > /tmp/docker-build-live.log 2>&1 & BUILD_PID=$!; echo "Build started, PID: $BUILD_PID"; tail -f /tmp/docker-build-live.log +``` + +**Why this works:** +- Build runs in background (`&`) - immune to terminal disconnects/Ctrl+C +- Output redirected to log file - can review later if needed +- `tail -f` shows real-time progress - see compilation, linting, errors +- Can Ctrl+C the `tail -f` without killing build - build continues +- Verification after: `tail -50 /tmp/docker-build-live.log` to check success + +**Success indicators:** +- `✓ Compiled successfully in 27s` +- `✓ Generating static pages (30/30)` +- `#22 naming to docker.io/library/traderv4-trading-bot done` +- `DONE X.Xs` on final step + +**Failure indicators:** +- `Failed to compile.` +- `Type error:` +- `ERROR: process "/bin/sh -c npm run build" did not complete successfully: exit code: 1` + +**After successful build:** +```bash +# Deploy new container +docker compose up -d --force-recreate trading-bot + +# Verify it started +docker logs --tail=30 trading-bot-v4 + +# Confirm deployed version +docker logs trading-bot-v4 | grep "Server starting" | head -1 +``` + +**DO NOT use:** `docker compose build trading-bot` in foreground - one network hiccup kills 60s of work --- ## Critical Components -### 1. Signal Quality Scoring (`lib/trading/signal-quality.ts`) +### 1. Phantom Trade Auto-Closure System +**Purpose:** Automatically close positions when size mismatch detected (position opened but wrong size) + +**When triggered:** +- Position opened on Drift successfully +- Expected size: $50 (50% @ 1x leverage) +- Actual size: $1.37 (7% fill - likely oracle price stale or exchange rejection) +- Size ratio < 50% threshold → phantom detected + +**Automated response (all happens in <1 second):** +1. **Immediate closure:** Market order closes 100% of phantom position +2. **Database logging:** Creates trade record with `status='phantom'`, saves P&L +3. **n8n notification:** Returns HTTP 200 with full details (not 500 - allows workflow to continue) +4. **Telegram alert:** Message includes entry/exit prices, P&L, reason, transaction IDs + +**Why auto-close instead of manual intervention:** +- User may be asleep, away from devices, unavailable for hours +- Unmonitored position = unlimited risk exposure +- Position Manager won't track phantom (by design) +- No TP/SL protection, no trailing stop, no monitoring +- Better to exit with small loss/gain than leave position exposed +- Re-entry always possible if setup was actually good + +**Example notification:** +``` +⚠️ PHANTOM TRADE AUTO-CLOSED + +Symbol: SOL-PERP +Direction: LONG +Expected Size: $48.75 +Actual Size: $1.37 (2.8%) + +Entry: $168.50 +Exit: $168.45 +P&L: -$0.02 + +Reason: Size mismatch detected - likely oracle price issue or exchange rejection +Action: Position auto-closed for safety (unmonitored positions = risk) + +TX: 5Yx2Fm8vQHKLdPaw... +``` + +**Database tracking:** +- `status='phantom'` field identifies these trades +- `isPhantom=true`, `phantomReason='ORACLE_PRICE_MISMATCH'` +- `expectedSizeUSD`, `actualSizeUSD` fields for analysis +- Exit reason: `'manual'` (phantom auto-close category) +- Enables post-trade analysis of phantom frequency and patterns + +**Code location:** `app/api/trading/execute/route.ts` lines 322-445 + +### 2. Signal Quality Scoring (`lib/trading/signal-quality.ts`) **Purpose:** Unified quality validation system that scores trading signals 0-100 based on 5 market metrics **Timeframe-aware thresholds:** @@ -991,6 +1104,40 @@ trade.realizedPnL += actualRealizedPnL // NOT: result.realizedPnL from SDK - **Impact:** 99% of transient DNS failures now auto-recover, preventing missed trades - **Documentation:** See `docs/DNS_RETRY_LOGIC.md` for monitoring queries and metrics +29. **Declaring fixes "working" before deployment (CRITICAL - Nov 13, 2025):** + - **Symptom:** AI says "position is protected" or "fix is deployed" when container still running old code + - **Root Cause:** Conflating "code committed to git" with "code running in production" + - **Real Incident:** Database-first fix committed 15:56, declared "working" at 19:42, but container started 15:06 (old code) + - **Result:** Unprotected position opened, database save failed silently, Position Manager never tracked it + - **Financial Impact:** User discovered $250+ unprotected position 3.5 hours after opening + - **Verification Required:** + ```bash + # ALWAYS check before declaring fix deployed: + docker logs trading-bot-v4 | grep "Server starting" | head -1 + # Compare container start time to git commit timestamp + # If container older: FIX NOT DEPLOYED + ``` + - **Rule:** NEVER say "fixed", "working", "protected", or "deployed" without verifying container restart timestamp + - **Impact:** This is a REAL MONEY system - premature declarations cause financial losses + - **Documentation:** Added mandatory deployment verification to VERIFICATION MANDATE section + +30. **Phantom trade notification workflow breaks (Nov 14, 2025):** + - **Symptom:** Phantom trade detected, position opened on Drift, but n8n workflow stops with HTTP 500 error. User NOT notified. + - **Root Cause:** Execute endpoint returned HTTP 500 when phantom detected, causing n8n chain to halt before Telegram notification + - **Problem:** Unmonitored phantom position on exchange while user is asleep/away = unlimited risk exposure + - **Fix:** Auto-close phantom trades immediately + return HTTP 200 with warning (allows n8n to continue) + ```typescript + // When phantom detected in app/api/trading/execute/route.ts: + // 1. Immediately close position via closePosition() + // 2. Save to database (create trade + update with exit info) + // 3. Return HTTP 200 with full notification message in response + // 4. n8n workflow continues to Telegram notification step + ``` + - **Response format change:** `{ success: true, warning: 'Phantom trade detected and auto-closed', isPhantom: true, message: '[Full notification text]', phantomDetails: {...} }` + - **Why auto-close:** User can't always respond (sleeping, no phone, traveling). Better to exit with small loss/gain than leave unmonitored position exposed. + - **Impact:** Protects user from unlimited risk during unavailable hours. Phantom trades are rare edge cases (oracle issues, exchange rejections). + - **Database tracking:** `status='phantom'`, `exitReason='manual'`, enables analysis of phantom frequency and patterns + ## File Conventions - **API routes:** `app/api/[feature]/[action]/route.ts` (Next.js 15 App Router) @@ -1118,6 +1265,12 @@ if (!enabled) { - Never assume SDK data format - log raw values to verify - SQL query with manual calculation to compare results - Test boundary cases: 0%, 100%, min/max values +11. **DEPLOYMENT VERIFICATION (MANDATORY):** Before declaring ANY fix working: + - Check container start time vs commit timestamp + - If container older than commit: CODE NOT DEPLOYED + - Restart container and verify new code is running + - Never say "fixed" or "protected" without deployment confirmation + - This is a REAL MONEY system - unverified fixes cause losses ## Development Roadmap diff --git a/app/api/trading/execute/route.ts b/app/api/trading/execute/route.ts index c198afe..27ebe1f 100644 --- a/app/api/trading/execute/route.ts +++ b/app/api/trading/execute/route.ts @@ -7,7 +7,7 @@ import { NextRequest, NextResponse } from 'next/server' import { initializeDriftService } from '@/lib/drift/client' -import { openPosition, placeExitOrders } from '@/lib/drift/orders' +import { openPosition, placeExitOrders, closePosition } from '@/lib/drift/orders' import { normalizeTradingViewSymbol } from '@/config/trading' import { getMergedConfig } from '@/config/trading' import { getInitializedPositionManager, ActiveTrade } from '@/lib/trading/position-manager' @@ -320,11 +320,42 @@ export async function POST(request: NextRequest): Promise {process.exit(r.statusCode === 200 ? 0 : 1)})"] + interval: 30s + timeout: 10s + retries: 3 + + # ================================ + # PostgreSQL Database (Optional) + # ================================ + postgres: + image: postgres:16-alpine + container_name: trading-bot-postgres + restart: unless-stopped + ports: + - "5432:5432" + environment: + POSTGRES_DB: trading_bot_v4 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=en_US.UTF-8" + volumes: + # Persist database data + - postgres-data:/var/lib/postgresql/data + + # Custom initialization scripts (optional) + - ./prisma/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + networks: + - trading-net + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + +# ================================ +# Networks +# ================================ +networks: + trading-net: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 + +# ================================ +# Volumes +# ================================ +volumes: + postgres-data: + driver: local diff --git a/lib/drift/client.ts b/lib/drift/client.ts index 9424654..9a52d01 100644 --- a/lib/drift/client.ts +++ b/lib/drift/client.ts @@ -102,8 +102,12 @@ export class DriftService { error?.code === 'EAI_AGAIN' || error?.cause?.code === 'EAI_AGAIN' + console.log(`🔍 Error detection: isTransient=${isTransient}, attempt=${attempt}/${maxRetries}`) + console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`) + if (!isTransient || attempt === maxRetries) { // Non-transient error or max retries reached - fail immediately + console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`) throw error }