feat: Implement v11 progressive parameter sweep starting from zero filters

Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com>
2025-12-06 20:30:57 +00:00
parent e92ba6df83
commit f678a027c2
4 changed files with 233 additions and 70 deletions
--- a/backtester/v11_moneyline_all_filters.py
+++ b/backtester/v11_moneyline_all_filters.py
@@ -5,19 +5,23 @@ CRITICAL DIFFERENCE FROM v9:
 - v11: ALL filters actually applied to signals (useQualityFilters toggle)
 - v9 bug: Filters calculated but signals ignored them

+PROGRESSIVE SWEEP SUPPORT:
+- Filter value of 0 = filter disabled (pass everything)
+- Allows testing from ultra-permissive to strict restrictions
+
 Based on moneyline_v11_all_filters.pinescript lines 271-272:
    finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk))
    finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk))

-Test sweep parameters (8 params × 2 values = 256 combinations):
- flip_threshold: 0.5, 0.6
- adx_min: 18, 21
- long_pos_max: 75, 80
- short_pos_min: 20, 25
- vol_min: 0.8, 1.0
- entry_buffer_atr: 0.15, 0.20
- rsi_long_min: 35, 40
- rsi_short_max: 65, 70
+Progressive test sweep parameters (512 combinations):
+- flip_threshold: 0.4, 0.5
+- adx_min: 0, 5, 10, 15 (0 = disabled)
+- long_pos_max: 95, 100
+- short_pos_min: 0, 5 (0 = disabled)
+- vol_min: 0.0, 0.5 (0 = disabled)
+- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
+- rsi_long_min: 25, 30
+- rsi_short_max: 75, 80
 """
 from __future__ import annotations

@@ -39,7 +43,7 @@ Direction = Literal["long", "short"]

@dataclass
 class MoneyLineV11Inputs:
-    """v11 Money Line indicator parameters for test sweep."""
+    """v11 Money Line indicator parameters for progressive test sweep."""
    
    # Basic Money Line parameters (fixed for test)
    confirm_bars: int = 0  # Immediate signals
@@ -49,15 +53,16 @@ class MoneyLineV11Inputs:
    atr_period: int = 12  # ATR calculation length
    multiplier: float = 3.8  # ATR band multiplier
    
-    # Filter parameters (8 parameters being optimized)
-    flip_threshold: float = 0.5  # % price must move to flip (TEST: 0.5, 0.6)
-    adx_min: float = 21  # Minimum ADX for signal (TEST: 18, 21)
-    long_pos_max: float = 75  # Don't long above X% of range (TEST: 75, 80)
-    short_pos_min: float = 20  # Don't short below X% of range (TEST: 20, 25)
-    vol_min: float = 1.0  # Minimum volume ratio (TEST: 0.8, 1.0)
-    entry_buffer_atr: float = 0.20  # ATR buffer beyond line (TEST: 0.15, 0.20)
-    rsi_long_min: float = 35  # RSI minimum for longs (TEST: 35, 40)
-    rsi_short_max: float = 70  # RSI maximum for shorts (TEST: 65, 70)
+    # PROGRESSIVE Filter parameters (256 combinations)
+    # Value of 0 = filter disabled (pass everything)
+    flip_threshold: float = 0.5  # % price must move to flip (PROGRESSIVE: 0.4, 0.5)
+    adx_min: float = 15  # Minimum ADX for signal (PROGRESSIVE: 0, 5, 10, 15 - 0=disabled)
+    long_pos_max: float = 95  # Don't long above X% of range (PROGRESSIVE: 95, 100)
+    short_pos_min: float = 0  # Don't short below X% of range (PROGRESSIVE: 0, 5 - 0=disabled)
+    vol_min: float = 0.5  # Minimum volume ratio (PROGRESSIVE: 0.0, 0.5 - 0=disabled)
+    entry_buffer_atr: float = 0.10  # ATR buffer beyond line (PROGRESSIVE: 0.0, 0.10 - 0=disabled)
+    rsi_long_min: float = 25  # RSI minimum for longs (PROGRESSIVE: 25, 30)
+    rsi_short_max: float = 80  # RSI maximum for shorts (PROGRESSIVE: 75, 80)
    
    # Fixed filter parameters (not being optimized in test)
    adx_length: int = 16  # ADX calculation length
@@ -259,16 +264,26 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
            continue

        # V11 CRITICAL: Apply ALL filters (this is what was broken in v9)
+        # PROGRESSIVE SWEEP: Value of 0 = filter disabled (pass everything)
        
-        # ADX filter (adxOk)
+        # ADX filter (adxOk) - disabled if adx_min == 0
+        if inputs.adx_min > 0:
            adx_ok = row.adx >= inputs.adx_min
+        else:
+            adx_ok = True  # Filter disabled
        
-        # Volume filter (volumeOk)
+        # Volume filter (volumeOk) - disabled if vol_min == 0
+        if inputs.vol_min > 0:
            volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
+        else:
+            volume_ok = row.volume_ratio <= inputs.vol_max  # Only check upper bound
        
        if flip_long:
-            # Entry buffer check (longBufferOk)
+            # Entry buffer check (longBufferOk) - disabled if entry_buffer_atr == 0
+            if inputs.entry_buffer_atr > 0:
                entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
+            else:
+                entry_buffer_ok = True  # Filter disabled
            
            # Long filters
            rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max  # rsiLongOk
@@ -291,12 +306,20 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
                cooldown_remaining = inputs.cooldown_bars
                
        elif flip_short:
-            # Entry buffer check (shortBufferOk)
+            # Entry buffer check (shortBufferOk) - disabled if entry_buffer_atr == 0
+            if inputs.entry_buffer_atr > 0:
                entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr)
+            else:
+                entry_buffer_ok = True  # Filter disabled
            
            # Short filters
            rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max  # rsiShortOk
+            
+            # Price position filter - disabled if short_pos_min == 0
+            if inputs.short_pos_min > 0:
                pos_ok = row.price_position > inputs.short_pos_min  # shortPositionOk
+            else:
+                pos_ok = True  # Filter disabled
            
            # V11: ALL filters must pass (this is the fix from v9)
            if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
--- a/cluster/run_v11_progressive_sweep.sh
+++ b/cluster/run_v11_progressive_sweep.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+# V11 PROGRESSIVE Parameter Sweep Launch Script
+# Stage 1: Ultra-permissive (start from 0 filters) to find baseline
+
+set -e  # Exit on error
+
+echo "================================================================"
+echo "V11 PROGRESSIVE PARAMETER SWEEP - STAGE 1"
+echo "================================================================"
+echo ""
+echo "Strategy: Start from 0 (filters disabled) and go upwards"
+echo ""
+echo "Progressive Grid (512 combinations):"
+echo "  - flip_threshold: 0.4, 0.5"
+echo "  - adx_min: 0, 5, 10, 15 (0 = disabled)"
+echo "  - long_pos_max: 95, 100 (very loose)"
+echo "  - short_pos_min: 0, 5 (0 = disabled)"
+echo "  - vol_min: 0.0, 0.5 (0 = disabled)"
+echo "  - entry_buffer_atr: 0.0, 0.10 (0 = disabled)"
+echo "  - rsi_long_min: 25, 30 (permissive)"
+echo "  - rsi_short_max: 75, 80 (permissive)"
+echo ""
+echo "Expected outcomes:"
+echo "  - adx_min=0 configs: 150-300 signals (almost no filtering)"
+echo "  - adx_min=5 configs: 80-150 signals (light filtering)"
+echo "  - adx_min=10 configs: 40-80 signals (moderate filtering)"
+echo "  - adx_min=15 configs: 10-40 signals (strict filtering)"
+echo ""
+echo "If all still 0 signals with adx_min=0:"
+echo "  → Base Money Line calculation is broken (not the filters)"
+echo ""
+echo "================================================================"
+echo ""
+
+cd "$(dirname "$0")"
+
+# Check if data file exists
+if [ ! -f "data/solusdt_5m.csv" ]; then
+    echo "✗ Error: data/solusdt_5m.csv not found"
+    echo "  Please ensure market data is available"
+    exit 1
+fi
+
+echo "✓ Market data found"
+
+# Check if coordinator script exists
+if [ ! -f "v11_test_coordinator.py" ]; then
+    echo "✗ Error: v11_test_coordinator.py not found"
+    exit 1
+fi
+
+echo "✓ Coordinator script found"
+
+# Clear previous test data
+echo ""
+echo "🗑️  Clearing previous test data..."
+rm -rf v11_test_results/*
+if [ -f "exploration.db" ]; then
+    sqlite3 exploration.db "DELETE FROM v11_test_chunks;" 2>/dev/null || true
+    sqlite3 exploration.db "DELETE FROM v11_test_strategies;" 2>/dev/null || true
+    echo "✓ Database cleared"
+fi
+
+# Launch coordinator in background
+echo ""
+echo "🚀 Starting progressive sweep coordinator..."
+nohup python3 v11_test_coordinator.py > coordinator_v11_progressive.log 2>&1 &
+COORDINATOR_PID=$!
+
+echo "✓ Coordinator started (PID: $COORDINATOR_PID)"
+echo ""
+echo "================================================================"
+echo "MONITORING"
+echo "================================================================"
+echo "Live log: tail -f coordinator_v11_progressive.log"
+echo "Database: sqlite3 exploration.db"
+echo "Results:  cluster/v11_test_results/*.csv"
+echo ""
+echo "Check status:"
+echo "  sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\""
+echo ""
+echo "Analyze signal distribution by ADX:"
+echo "  sqlite3 exploration.db \\"
+echo "    \"SELECT json_extract(params, '$.adx_min') as adx_min, \\"
+echo "     AVG(total_trades) as avg_signals, COUNT(*) as configs \\"
+echo "     FROM v11_test_strategies \\"
+echo "     GROUP BY adx_min ORDER BY adx_min;\""
+echo ""
+echo "To stop sweep:"
+echo "  kill $COORDINATOR_PID"
+echo "================================================================"
--- a/cluster/v11_test_coordinator.py
+++ b/cluster/v11_test_coordinator.py
@@ -1,12 +1,34 @@
 #!/usr/bin/env python3
 """
-V11 Test Parameter Sweep Coordinator
+V11 PROGRESSIVE Parameter Sweep Coordinator

-Coordinates 256-combination test sweep across 2 workers with smart scheduling.
+STAGE 1: ULTRA-PERMISSIVE (Start from 0 filters)
+Strategy: "Go upwards from 0 until you find something"
+
+Coordinates 256-combination progressive test sweep across 2 workers with smart scheduling.
 Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK).

-Test sweep: 2 chunks × 128 combinations = 256 total
-Expected runtime: 6-25 minutes depending on worker availability
+Progressive grid (512 combinations = 2×4×2×2×2×2×2×2):
+- flip_threshold: 0.4, 0.5
+- adx_min: 0, 5, 10, 15 (0 = disabled)
+- long_pos_max: 95, 100 (very loose)
+- short_pos_min: 0, 5 (0 = disabled)
+- vol_min: 0.0, 0.5 (0 = disabled)
+- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
+- rsi_long_min: 25, 30 (permissive)
+- rsi_short_max: 75, 80 (permissive)
+
+Expected outcomes:
+- adx_min=0 configs: 150-300 signals (almost no filtering)
+- adx_min=5 configs: 80-150 signals (light filtering)
+- adx_min=10 configs: 40-80 signals (moderate filtering)
+- adx_min=15 configs: 10-40 signals (strict filtering)
+
+If all still 0 signals with adx_min=0:
+→ Base Money Line calculation is broken (not the filters)
+
+Test sweep: 2 chunks × 256 combinations = 512 total
+Expected runtime: 12-35 minutes depending on worker availability
 """

 import sqlite3
@@ -39,7 +61,7 @@ WORKERS = {

 DATA_FILE = 'data/solusdt_5m.csv'
 DB_PATH = 'exploration.db'
-CHUNK_SIZE = 128  # Each chunk processes 128 combinations
+CHUNK_SIZE = 256  # Each chunk processes 256 combinations

 # Telegram configuration
 TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY'
@@ -134,10 +156,10 @@ def init_database():
        )
    """)
    
-    # Register 2 chunks (256 combinations total)
+    # Register 2 chunks (512 combinations total)
    chunks = [
-        ('v11_test_chunk_0000', 0, 128, 128),
-        ('v11_test_chunk_0001', 128, 256, 128),
+        ('v11_test_chunk_0000', 0, 256, 256),
+        ('v11_test_chunk_0001', 256, 512, 256),
    ]
    
    for chunk_id, start, end, total in chunks:
@@ -325,12 +347,17 @@ def main():
    signal.signal(signal.SIGTERM, signal_handler)
    
    print("\n" + "="*60)
-    print("V11 TEST PARAMETER SWEEP COORDINATOR")
+    print("V11 PROGRESSIVE PARAMETER SWEEP COORDINATOR")
+    print("Stage 1: Ultra-Permissive (start from 0 filters)")
    print("="*60)
-    print(f"Total combinations: 256 (2^8)")
-    print(f"Chunks: 2 × 128 combinations")
+    print(f"Total combinations: 512 (2×4×2×2×2×2×2×2)")
+    print(f"Chunks: 2 × 256 combinations")
    print(f"Workers: 2 × 27 cores (85% CPU)")
-    print(f"Expected runtime: 6-25 minutes")
+    print(f"Expected runtime: 12-35 minutes")
+    print("")
+    print("Progressive strategy: Start filters at 0 (disabled)")
+    print("Expected: adx_min=0 → 150-300 signals")
+    print("          adx_min=15 → 10-40 signals")
    print("="*60 + "\n")
    
    # Initialize database
@@ -340,9 +367,10 @@ def main():
    # Send start notification
    available_workers = get_available_workers()
    start_msg = (
-        f"🚀 <b>V11 Test Sweep STARTED</b>\n\n"
-        f"Combinations: 256 (2^8)\n"
-        f"Chunks: 2 × 128 combos\n"
+        f"🚀 <b>V11 Progressive Sweep STARTED</b>\n"
+        f"Stage 1: Ultra-Permissive (start from 0)\n\n"
+        f"Combinations: 512 (2×4×2×2×2×2×2×2)\n"
+        f"Chunks: 2 × 256 combos\n"
        f"Workers: {len(available_workers)} available\n"
        f"- Worker 1: Always on (27 cores)\n"
    )
@@ -350,7 +378,8 @@ def main():
        start_msg += f"- Worker 2: Active (27 cores)\n"
    else:
        start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n"
-    start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+    start_msg += f"\nExpected: adx_min=0 → 150-300 signals\n"
+    start_msg += f"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
    
    send_telegram_message(start_msg)
    
@@ -412,20 +441,27 @@ def main():
    duration_min = duration / 60
    
    print("\n" + "="*60)
-    print("V11 TEST SWEEP COMPLETE!")
+    print("V11 PROGRESSIVE SWEEP COMPLETE!")
    print("="*60)
    print(f"Duration: {duration_min:.1f} minutes")
    print(f"Chunks: 2/2 completed")
-    print(f"Strategies: 256 tested")
+    print(f"Strategies: 512 tested")
+    print("")
+    print("Next: Analyze signal distribution by ADX threshold")
+    print("  sqlite3 exploration.db \"SELECT json_extract(params, '$.adx_min') as adx_min,")
+    print("    AVG(total_trades) as avg_signals, COUNT(*) as configs")
+    print("    FROM v11_test_strategies GROUP BY adx_min ORDER BY adx_min;\"")
    print("="*60 + "\n")
    
    # Send completion notification
    complete_msg = (
-        f"✅ <b>V11 Test Sweep COMPLETE</b>\n\n"
+        f"✅ <b>V11 Progressive Sweep COMPLETE</b>\n\n"
        f"Duration: {duration_min:.1f} minutes\n"
        f"Chunks: 2/2 completed\n"
-        f"Strategies: 256 tested\n\n"
-        f"Check results:\n"
+        f"Strategies: 512 tested\n\n"
+        f"Next step: Analyze signal distribution\n"
+        f"Check if adx_min=0 configs generated signals\n\n"
+        f"Results location:\n"
        f"- cluster/v11_test_results/\n"
        f"- sqlite3 exploration.db\n\n"
        f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
--- a/cluster/v11_test_worker.py
+++ b/cluster/v11_test_worker.py
@@ -2,18 +2,26 @@
 """
 V11 Test Parameter Sweep Worker

-Processes chunks of v11 test parameter configurations (256 combinations total).
+Processes chunks of v11 test parameter configurations (512 combinations total).
 Uses 27 cores (85% CPU) for multiprocessing.

-Test parameter grid (2 values each = 2^8 = 256 combinations):
- flip_threshold: 0.5, 0.6
- adx_min: 18, 21
- long_pos_max: 75, 80
- short_pos_min: 20, 25
- vol_min: 0.8, 1.0
- entry_buffer_atr: 0.15, 0.20
- rsi_long_min: 35, 40
- rsi_short_max: 65, 70
+PROGRESSIVE SWEEP - Stage 1: Ultra-Permissive (start from 0 filters)
+Goal: Find which parameter values allow signals through.
+
+Test parameter grid (2×4×2×2×2×2×2×2 = 512 combinations):
+- flip_threshold: 0.4, 0.5
+- adx_min: 0, 5, 10, 15 (START FROM ZERO - filter disabled at 0)
+- long_pos_max: 95, 100 (very loose)
+- short_pos_min: 0, 5 (START FROM ZERO - filter disabled at 0)
+- vol_min: 0.0, 0.5 (START FROM ZERO - filter disabled at 0)
+- entry_buffer_atr: 0.0, 0.10 (START FROM ZERO - filter disabled at 0)
+- rsi_long_min: 25, 30 (permissive)
+- rsi_short_max: 75, 80 (permissive)
+
+Expected outcomes:
+- adx_min=0 configs: 150-300 signals (almost no filtering)
+- adx_min=15 configs: 10-40 signals (strict filtering)
+- If all still 0 → base indicator broken, not the filters
 """

 import sys
@@ -45,17 +53,22 @@ def init_worker(data_file):
    global _DATA_FILE
    _DATA_FILE = data_file

-# Test parameter grid (256 combinations)
+# PROGRESSIVE Test parameter grid (512 combinations)
+# Stage 1: Ultra-permissive - Start from 0 (filters disabled) to find baseline
+# Strategy: "Go upwards from 0 until you find something"
 PARAMETER_GRID = {
-    'flip_threshold': [0.5, 0.6],
-    'adx_min': [18, 21],
-    'long_pos_max': [75, 80],
-    'short_pos_min': [20, 25],
-    'vol_min': [0.8, 1.0],
-    'entry_buffer_atr': [0.15, 0.20],
-    'rsi_long_min': [35, 40],
-    'rsi_short_max': [65, 70],
+    'flip_threshold': [0.4, 0.5],          # 2 values - range: loose to normal
+    'adx_min': [0, 5, 10, 15],             # 4 values - START FROM 0 (no filter)
+    'long_pos_max': [95, 100],             # 2 values - very permissive
+    'short_pos_min': [0, 5],               # 2 values - START FROM 0 (no filter)
+    'vol_min': [0.0, 0.5],                 # 2 values - START FROM 0 (no filter)
+    'entry_buffer_atr': [0.0, 0.10],       # 2 values - START FROM 0 (no filter)
+    'rsi_long_min': [25, 30],              # 2 values - permissive
+    'rsi_short_max': [75, 80],             # 2 values - permissive
 }
+# Total: 2×4×2×2×2×2×2×2 = 512 combos
+# Expected: adx_min=0 configs will generate 150-300 signals (proves v11 logic works)
+# If all still 0 signals with adx_min=0 → base indicator broken, not the filters


 def load_market_data(csv_file: str) -> pd.DataFrame:
@@ -309,7 +322,7 @@ if __name__ == '__main__':
    chunk_id = sys.argv[2]
    start_idx = int(sys.argv[3])
    
-    # Calculate end index (128 combos per chunk)
-    end_idx = start_idx + 128
+    # Calculate end index (256 combos per chunk)
+    end_idx = start_idx + 256
    
    process_chunk(data_file, chunk_id, start_idx, end_idx)