#!/usr/bin/env python3
"""
Backtest Q≥95 + Instant Reversal + HTF + 5-Candle Exit Strategy

Replays historical 5m trades and applies new filters to validate profitability.
"""

import os
import sys
from datetime import datetime, timedelta
from decimal import Decimal
from typing import Dict, List, Optional, Tuple
import psycopg2
from psycopg2.extras import RealDictCursor

# Configuration
DB_CONFIG = {
    'host': 'localhost',
    'port': 5432,
    'database': 'trading_bot_v4',
    'user': 'postgres',
    'password': os.getenv('POSTGRES_PASSWORD', 'postgres')
}

STARTING_CAPITAL = Decimal('97.55')
Q_THRESHOLD = 95
HTF_Q_THRESHOLD = 85  # If Q < 85 and same direction as 15m, block
INSTANT_REVERSAL_ATR_MULTIPLIER = 0.5  # Prior candle body > k*ATR
TIME_EXIT_CANDLES = 5  # Exit after 5 candles if MFE < threshold
TIME_EXIT_MFE_THRESHOLD = Decimal('30.00')  # Minimum MFE to avoid time exit
FEE_RATE = Decimal('0.0004')  # 0.04% taker fee
SLIPPAGE_RATE = Decimal('0.005')  # 0.5% slippage estimate


class Trade:
    """Represents a historical trade with all metadata"""
    def __init__(self, row: Dict):
        self.id = row['id']
        self.entry_time = row['entryTime']
        self.exit_time = row['exitTime']
        self.direction = row['direction']
        self.timeframe = row['timeframe']
        self.quality_score = row['signalQualityScore']
        self.entry_price = Decimal(str(row['entryPrice']))
        self.exit_price = Decimal(str(row['exitPrice'])) if row['exitPrice'] else None
        self.exit_reason = row['exitReason']
        self.realized_pnl = Decimal(str(row['realizedPnL'])) if row['realizedPnL'] else Decimal('0')
        self.mfe = Decimal(str(row['maxFavorableExcursion'])) if row['maxFavorableExcursion'] else Decimal('0')
        self.mae = Decimal(str(row['maxAdverseExcursion'])) if row['maxAdverseExcursion'] else Decimal('0')
        self.size = Decimal(str(row['size'])) if row.get('size') else Decimal('1')
        self.atr = Decimal(str(row['atr'])) if row.get('atr') else None
        
        # Calculate duration in candles (5m = 300s)
        if self.exit_time:
            duration_seconds = (self.exit_time - self.entry_time).total_seconds()
            self.candles_in_trade = max(1, int(duration_seconds / 300))
        else:
            self.candles_in_trade = 0
        
        # Filter flags (will be set during backtest)
        self.htf_blocked = False
        self.instant_reversal_detected = False
        self.would_enter = False
        self.simulated_pnl = Decimal('0')
        self.simulated_exit_reason = None


def get_db_connection():
    """Connect to Postgres database"""
    return psycopg2.connect(**DB_CONFIG, cursor_factory=RealDictCursor)


def fetch_historical_trades(conn, start_date: str = '2024-11-19') -> List[Trade]:
    """Fetch all 5m trades since start_date"""
    query = """
    SELECT 
        t.id,
        t."entryTime",
        t."exitTime",
        t.direction,
        t.timeframe,
        t."signalQualityScore",
        t."entryPrice",
        t."exitPrice",
        t."exitReason",
        t."realizedPnL",
        t."maxFavorableExcursion",
        t."maxAdverseExcursion",
        t.size,
        t.metadata->>'atr' as atr
    FROM "Trade" t
    WHERE t.timeframe = '5'
      AND t."signalSource" != 'manual'
      AND t.status = 'closed'
      AND t."exitReason" IS NOT NULL
      AND t."entryTime" >= %s
    ORDER BY t."entryTime" ASC
    """
    
    with conn.cursor() as cur:
        cur.execute(query, (start_date,))
        rows = cur.fetchall()
    
    return [Trade(row) for row in rows]


def fetch_htf_signal(conn, entry_time: datetime) -> Optional[str]:
    """Fetch most recent 15m BlockedSignal direction before entry_time"""
    query = """
    SELECT direction
    FROM "BlockedSignal"
    WHERE timeframe = '15'
      AND "createdAt" <= %s
    ORDER BY "createdAt" DESC
    LIMIT 1
    """
    
    with conn.cursor() as cur:
        cur.execute(query, (entry_time,))
        row = cur.fetchone()
        return row['direction'] if row else None


def detect_instant_reversal(trade: Trade, atr_multiplier: float = INSTANT_REVERSAL_ATR_MULTIPLIER) -> bool:
    """
    Detect if prior candle was a strong reversal against trade direction.
    
    Heuristic (without actual candle data):
    - If trade hit SL within 1 candle and MAE is large relative to entry, flag as instant reversal.
    - Ideally would check prior candle body vs ATR, but we approximate from MAE.
    """
    if trade.exit_reason == 'SL' and trade.candles_in_trade <= 1:
        # Fast stop-out suggests instant reversal
        if trade.atr:
            # Check if MAE > threshold * ATR (strong move against us immediately)
            threshold = Decimal(str(atr_multiplier)) * trade.atr
            if abs(trade.mae) > threshold:
                return True
        else:
            # Fallback: any SL within 1 candle is considered instant reversal
            return True
    
    return False


def apply_htf_filter(trade: Trade, htf_direction: Optional[str]) -> bool:
    """
    Check if trade should be blocked by HTF filter.
    Block if: 5m direction == 15m direction AND quality < HTF_Q_THRESHOLD
    """
    if not htf_direction:
        return False  # No HTF data, don't block
    
    if trade.direction == htf_direction and trade.quality_score < HTF_Q_THRESHOLD:
        return True
    
    return False


def simulate_exit_with_time_limit(trade: Trade) -> Tuple[Decimal, str]:
    """
    Simulate exit with 5-candle time limit.
    
    Rules:
    - If exited via TP1/TP2/TRAILING_SL: use realized PnL
    - If MFE >= TIME_EXIT_MFE_THRESHOLD: use realized PnL
    - If candles <= TIME_EXIT_CANDLES: use realized PnL
    - Else (long trade, low MFE): simulate exit at MFE * 0.5 or MAE (whichever is better)
    
    Apply fees/slippage to simulated exits.
    """
    # Natural exits (already good)
    if trade.exit_reason in ['TP1', 'TP2', 'TRAILING_SL']:
        return trade.realized_pnl, trade.exit_reason
    
    # High MFE achieved - keep realized PnL
    if trade.mfe >= TIME_EXIT_MFE_THRESHOLD:
        return trade.realized_pnl, trade.exit_reason
    
    # Exited within time window - keep realized PnL
    if trade.candles_in_trade <= TIME_EXIT_CANDLES:
        return trade.realized_pnl, trade.exit_reason
    
    # Simulate time exit: take 50% of MFE or MAE, whichever is better
    # This approximates exiting after 5 candles if trade didn't hit targets
    simulated_gross = max(trade.mfe * Decimal('0.5'), trade.mae)
    
    # Apply costs (fees + slippage on entry and exit)
    entry_cost = trade.entry_price * trade.size * (FEE_RATE + SLIPPAGE_RATE)
    exit_cost = trade.entry_price * trade.size * (FEE_RATE + SLIPPAGE_RATE)
    total_cost = entry_cost + exit_cost
    
    simulated_pnl = simulated_gross - total_cost
    
    return simulated_pnl, 'TIME_EXIT_5_CANDLE'


def run_backtest(trades: List[Trade], conn) -> Dict:
    """
    Apply all filters and simulate new strategy.
    
    Returns metrics dict with comparison.
    """
    # Original strategy metrics (all trades)
    original_trades = trades
    original_count = len(original_trades)
    original_pnl = sum(t.realized_pnl for t in original_trades)
    original_wins = sum(1 for t in original_trades if t.realized_pnl > 0)
    original_hit_rate = (original_wins / original_count * 100) if original_count > 0 else 0
    
    # Apply new strategy filters
    qualified_trades = []
    
    for trade in trades:
        # Fetch HTF signal
        htf_direction = fetch_htf_signal(conn, trade.entry_time)
        
        # Check HTF filter
        trade.htf_blocked = apply_htf_filter(trade, htf_direction)
        
        # Check instant reversal
        trade.instant_reversal_detected = detect_instant_reversal(trade)
        
        # Check quality threshold
        quality_pass = trade.quality_score >= Q_THRESHOLD
        
        # Determine if would enter
        trade.would_enter = (
            quality_pass
            and not trade.htf_blocked
            and not trade.instant_reversal_detected
        )
        
        if trade.would_enter:
            # Simulate exit with time limit
            trade.simulated_pnl, trade.simulated_exit_reason = simulate_exit_with_time_limit(trade)
            qualified_trades.append(trade)
    
    # New strategy metrics
    new_count = len(qualified_trades)
    new_pnl = sum(t.simulated_pnl for t in qualified_trades)
    new_wins = sum(1 for t in qualified_trades if t.simulated_pnl > 0)
    new_hit_rate = (new_wins / new_count * 100) if new_count > 0 else 0
    
    # Filter breakdown
    blocked_by_q = sum(1 for t in trades if t.quality_score < Q_THRESHOLD)
    blocked_by_htf = sum(1 for t in trades if t.htf_blocked and t.quality_score >= Q_THRESHOLD)
    blocked_by_instant_reversal = sum(
        1 for t in trades 
        if t.instant_reversal_detected 
        and not t.htf_blocked 
        and t.quality_score >= Q_THRESHOLD
    )
    
    # Calculate date range
    if trades:
        start_date = min(t.entry_time for t in trades)
        end_date = max(t.entry_time for t in trades)
        days = (end_date - start_date).days
    else:
        days = 0
    
    return {
        'original': {
            'trades': original_count,
            'wins': original_wins,
            'hit_rate': original_hit_rate,
            'total_pnl': original_pnl,
            'ending_capital': STARTING_CAPITAL + original_pnl,
            'return_pct': (original_pnl / STARTING_CAPITAL * 100) if STARTING_CAPITAL > 0 else 0,
        },
        'new': {
            'trades': new_count,
            'wins': new_wins,
            'hit_rate': new_hit_rate,
            'total_pnl': new_pnl,
            'ending_capital': STARTING_CAPITAL + new_pnl,
            'return_pct': (new_pnl / STARTING_CAPITAL * 100) if STARTING_CAPITAL > 0 else 0,
        },
        'filters': {
            'blocked_by_q': blocked_by_q,
            'blocked_by_htf': blocked_by_htf,
            'blocked_by_instant_reversal': blocked_by_instant_reversal,
        },
        'meta': {
            'start_date': start_date if trades else None,
            'end_date': end_date if trades else None,
            'days': days,
            'daily_return_original': (original_pnl / STARTING_CAPITAL / days * 100) if days > 0 else 0,
            'daily_return_new': (new_pnl / STARTING_CAPITAL / days * 100) if days > 0 else 0,
        },
        'qualified_trades': qualified_trades,
    }


def print_report(results: Dict):
    """Print formatted backtest report"""
    print("\n" + "="*80)
    print("BACKTEST REPORT: Q≥95 + Instant Reversal + HTF + 5-Candle Exit")
    print("="*80)
    
    meta = results['meta']
    print(f"\nPeriod: {meta['start_date'].date()} to {meta['end_date'].date()} ({meta['days']} days)")
    print(f"Starting Capital: ${STARTING_CAPITAL}")
    
    print("\n" + "-"*80)
    print("ORIGINAL STRATEGY (Current Live)")
    print("-"*80)
    orig = results['original']
    print(f"Trades:          {orig['trades']}")
    print(f"Wins:            {orig['wins']} ({orig['hit_rate']:.1f}%)")
    print(f"Total PnL:       ${orig['total_pnl']:.2f}")
    print(f"Ending Capital:  ${orig['ending_capital']:.2f}")
    print(f"Total Return:    {orig['return_pct']:.2f}%")
    print(f"Daily Return:    {meta['daily_return_original']:.3f}%")
    
    print("\n" + "-"*80)
    print("NEW STRATEGY (Q≥95 + Filters)")
    print("-"*80)
    new = results['new']
    print(f"Trades:          {new['trades']}")
    print(f"Wins:            {new['wins']} ({new['hit_rate']:.1f}%)")
    print(f"Total PnL:       ${new['total_pnl']:.2f}")
    print(f"Ending Capital:  ${new['ending_capital']:.2f}")
    print(f"Total Return:    {new['return_pct']:.2f}%")
    print(f"Daily Return:    {meta['daily_return_new']:.3f}%")
    
    print("\n" + "-"*80)
    print("FILTER IMPACT")
    print("-"*80)
    filters = results['filters']
    print(f"Blocked by Q<{Q_THRESHOLD}:           {filters['blocked_by_q']} trades")
    print(f"Blocked by HTF:                      {filters['blocked_by_htf']} trades")
    print(f"Blocked by Instant Reversal:         {filters['blocked_by_instant_reversal']} trades")
    print(f"Total Filtered Out:                  {orig['trades'] - new['trades']} trades")
    
    print("\n" + "-"*80)
    print("COMPARISON")
    print("-"*80)
    pnl_delta = new['total_pnl'] - orig['total_pnl']
    return_delta = new['return_pct'] - orig['return_pct']
    trade_delta = new['trades'] - orig['trades']
    
    print(f"PnL Change:      ${pnl_delta:+.2f} ({'+' if pnl_delta >= 0 else ''}{return_delta:.2f}%)")
    print(f"Trade Count:     {trade_delta:+d} ({new['trades']} vs {orig['trades']})")
    print(f"Hit Rate Change: {new['hit_rate'] - orig['hit_rate']:+.1f}%")
    
    if new['trades'] > 0:
        avg_pnl_per_trade = new['total_pnl'] / new['trades']
        print(f"Avg PnL/Trade:   ${avg_pnl_per_trade:.2f}")
    
    print("\n" + "="*80)
    
    # Recommendation
    print("\nRECOMMENDATION:")
    if new['total_pnl'] > 0 and new['hit_rate'] >= 40 and new['trades'] >= 10:
        print("✓ NEW STRATEGY LOOKS PROMISING")
        print(f"  - Positive PnL (${new['total_pnl']:.2f})")
        print(f"  - Decent hit rate ({new['hit_rate']:.1f}%)")
        print(f"  - Sufficient trades ({new['trades']})")
        print("  → Consider implementing with shadow mode first")
    elif new['total_pnl'] > orig['total_pnl']:
        print("⚠ NEW STRATEGY SHOWS IMPROVEMENT BUT NEEDS MORE DATA")
        print(f"  - Better PnL (${pnl_delta:+.2f})")
        print(f"  - Only {new['trades']} trades (need ≥100 for confidence)")
        print("  → Run shadow mode for 4-6 weeks before live deployment")
    else:
        print("✗ NEW STRATEGY DOES NOT IMPROVE RESULTS")
        print(f"  - PnL worse by ${-pnl_delta:.2f}")
        print("  → Do not deploy; revisit filter thresholds")
    
    print("="*80 + "\n")


def main():
    """Main backtest execution"""
    print("Connecting to database...")
    conn = get_db_connection()
    
    try:
        print("Fetching historical trades...")
        trades = fetch_historical_trades(conn, start_date='2024-11-19')
        print(f"Loaded {len(trades)} trades")
        
        if not trades:
            print("No trades found. Exiting.")
            return
        
        print("\nRunning backtest...")
        results = run_backtest(trades, conn)
        
        print_report(results)
        
        # Optionally print sample qualified trades
        print("\nSAMPLE QUALIFIED TRADES (First 10):")
        print("-"*80)
        for i, trade in enumerate(results['qualified_trades'][:10], 1):
            print(f"{i}. {trade.entry_time.date()} | {trade.direction.upper():5} | "
                  f"Q={trade.quality_score} | Exit: {trade.simulated_exit_reason:20} | "
                  f"PnL: ${trade.simulated_pnl:+7.2f}")
        
        if len(results['qualified_trades']) > 10:
            print(f"... and {len(results['qualified_trades']) - 10} more")
        
    finally:
        conn.close()


if __name__ == '__main__':
    main()