From f678a027c278c55acf10c1e8a275635ebaf3e261 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 6 Dec 2025 20:30:57 +0000 Subject: [PATCH] feat: Implement v11 progressive parameter sweep starting from zero filters Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com> --- backtester/v11_moneyline_all_filters.py | 79 +++++++++++++-------- cluster/run_v11_progressive_sweep.sh | 91 +++++++++++++++++++++++++ cluster/v11_test_coordinator.py | 78 +++++++++++++++------ cluster/v11_test_worker.py | 55 +++++++++------ 4 files changed, 233 insertions(+), 70 deletions(-) create mode 100755 cluster/run_v11_progressive_sweep.sh diff --git a/backtester/v11_moneyline_all_filters.py b/backtester/v11_moneyline_all_filters.py index cb9c447..31deaed 100644 --- a/backtester/v11_moneyline_all_filters.py +++ b/backtester/v11_moneyline_all_filters.py @@ -5,19 +5,23 @@ CRITICAL DIFFERENCE FROM v9: - v11: ALL filters actually applied to signals (useQualityFilters toggle) - v9 bug: Filters calculated but signals ignored them +PROGRESSIVE SWEEP SUPPORT: +- Filter value of 0 = filter disabled (pass everything) +- Allows testing from ultra-permissive to strict restrictions + Based on moneyline_v11_all_filters.pinescript lines 271-272: finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk)) finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk)) -Test sweep parameters (8 params × 2 values = 256 combinations): -- flip_threshold: 0.5, 0.6 -- adx_min: 18, 21 -- long_pos_max: 75, 80 -- short_pos_min: 20, 25 -- vol_min: 0.8, 1.0 -- entry_buffer_atr: 0.15, 0.20 -- rsi_long_min: 35, 40 -- rsi_short_max: 65, 70 +Progressive test sweep parameters (512 combinations): +- flip_threshold: 0.4, 0.5 +- adx_min: 0, 5, 10, 15 (0 = disabled) +- long_pos_max: 95, 100 +- short_pos_min: 0, 5 (0 = disabled) +- vol_min: 0.0, 0.5 (0 = disabled) +- entry_buffer_atr: 0.0, 0.10 (0 = disabled) +- rsi_long_min: 25, 30 +- rsi_short_max: 75, 80 """ from __future__ import annotations @@ -39,7 +43,7 @@ Direction = Literal["long", "short"] @dataclass class MoneyLineV11Inputs: - """v11 Money Line indicator parameters for test sweep.""" + """v11 Money Line indicator parameters for progressive test sweep.""" # Basic Money Line parameters (fixed for test) confirm_bars: int = 0 # Immediate signals @@ -49,15 +53,16 @@ class MoneyLineV11Inputs: atr_period: int = 12 # ATR calculation length multiplier: float = 3.8 # ATR band multiplier - # Filter parameters (8 parameters being optimized) - flip_threshold: float = 0.5 # % price must move to flip (TEST: 0.5, 0.6) - adx_min: float = 21 # Minimum ADX for signal (TEST: 18, 21) - long_pos_max: float = 75 # Don't long above X% of range (TEST: 75, 80) - short_pos_min: float = 20 # Don't short below X% of range (TEST: 20, 25) - vol_min: float = 1.0 # Minimum volume ratio (TEST: 0.8, 1.0) - entry_buffer_atr: float = 0.20 # ATR buffer beyond line (TEST: 0.15, 0.20) - rsi_long_min: float = 35 # RSI minimum for longs (TEST: 35, 40) - rsi_short_max: float = 70 # RSI maximum for shorts (TEST: 65, 70) + # PROGRESSIVE Filter parameters (256 combinations) + # Value of 0 = filter disabled (pass everything) + flip_threshold: float = 0.5 # % price must move to flip (PROGRESSIVE: 0.4, 0.5) + adx_min: float = 15 # Minimum ADX for signal (PROGRESSIVE: 0, 5, 10, 15 - 0=disabled) + long_pos_max: float = 95 # Don't long above X% of range (PROGRESSIVE: 95, 100) + short_pos_min: float = 0 # Don't short below X% of range (PROGRESSIVE: 0, 5 - 0=disabled) + vol_min: float = 0.5 # Minimum volume ratio (PROGRESSIVE: 0.0, 0.5 - 0=disabled) + entry_buffer_atr: float = 0.10 # ATR buffer beyond line (PROGRESSIVE: 0.0, 0.10 - 0=disabled) + rsi_long_min: float = 25 # RSI minimum for longs (PROGRESSIVE: 25, 30) + rsi_short_max: float = 80 # RSI maximum for shorts (PROGRESSIVE: 75, 80) # Fixed filter parameters (not being optimized in test) adx_length: int = 16 # ADX calculation length @@ -259,16 +264,26 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs continue # V11 CRITICAL: Apply ALL filters (this is what was broken in v9) + # PROGRESSIVE SWEEP: Value of 0 = filter disabled (pass everything) - # ADX filter (adxOk) - adx_ok = row.adx >= inputs.adx_min + # ADX filter (adxOk) - disabled if adx_min == 0 + if inputs.adx_min > 0: + adx_ok = row.adx >= inputs.adx_min + else: + adx_ok = True # Filter disabled - # Volume filter (volumeOk) - volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max + # Volume filter (volumeOk) - disabled if vol_min == 0 + if inputs.vol_min > 0: + volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max + else: + volume_ok = row.volume_ratio <= inputs.vol_max # Only check upper bound if flip_long: - # Entry buffer check (longBufferOk) - entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr) + # Entry buffer check (longBufferOk) - disabled if entry_buffer_atr == 0 + if inputs.entry_buffer_atr > 0: + entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr) + else: + entry_buffer_ok = True # Filter disabled # Long filters rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max # rsiLongOk @@ -291,12 +306,20 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs cooldown_remaining = inputs.cooldown_bars elif flip_short: - # Entry buffer check (shortBufferOk) - entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr) + # Entry buffer check (shortBufferOk) - disabled if entry_buffer_atr == 0 + if inputs.entry_buffer_atr > 0: + entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr) + else: + entry_buffer_ok = True # Filter disabled # Short filters rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max # rsiShortOk - pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk + + # Price position filter - disabled if short_pos_min == 0 + if inputs.short_pos_min > 0: + pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk + else: + pos_ok = True # Filter disabled # V11: ALL filters must pass (this is the fix from v9) if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok: diff --git a/cluster/run_v11_progressive_sweep.sh b/cluster/run_v11_progressive_sweep.sh new file mode 100755 index 0000000..94b1ab2 --- /dev/null +++ b/cluster/run_v11_progressive_sweep.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# V11 PROGRESSIVE Parameter Sweep Launch Script +# Stage 1: Ultra-permissive (start from 0 filters) to find baseline + +set -e # Exit on error + +echo "================================================================" +echo "V11 PROGRESSIVE PARAMETER SWEEP - STAGE 1" +echo "================================================================" +echo "" +echo "Strategy: Start from 0 (filters disabled) and go upwards" +echo "" +echo "Progressive Grid (512 combinations):" +echo " - flip_threshold: 0.4, 0.5" +echo " - adx_min: 0, 5, 10, 15 (0 = disabled)" +echo " - long_pos_max: 95, 100 (very loose)" +echo " - short_pos_min: 0, 5 (0 = disabled)" +echo " - vol_min: 0.0, 0.5 (0 = disabled)" +echo " - entry_buffer_atr: 0.0, 0.10 (0 = disabled)" +echo " - rsi_long_min: 25, 30 (permissive)" +echo " - rsi_short_max: 75, 80 (permissive)" +echo "" +echo "Expected outcomes:" +echo " - adx_min=0 configs: 150-300 signals (almost no filtering)" +echo " - adx_min=5 configs: 80-150 signals (light filtering)" +echo " - adx_min=10 configs: 40-80 signals (moderate filtering)" +echo " - adx_min=15 configs: 10-40 signals (strict filtering)" +echo "" +echo "If all still 0 signals with adx_min=0:" +echo " → Base Money Line calculation is broken (not the filters)" +echo "" +echo "================================================================" +echo "" + +cd "$(dirname "$0")" + +# Check if data file exists +if [ ! -f "data/solusdt_5m.csv" ]; then + echo "✗ Error: data/solusdt_5m.csv not found" + echo " Please ensure market data is available" + exit 1 +fi + +echo "✓ Market data found" + +# Check if coordinator script exists +if [ ! -f "v11_test_coordinator.py" ]; then + echo "✗ Error: v11_test_coordinator.py not found" + exit 1 +fi + +echo "✓ Coordinator script found" + +# Clear previous test data +echo "" +echo "🗑️ Clearing previous test data..." +rm -rf v11_test_results/* +if [ -f "exploration.db" ]; then + sqlite3 exploration.db "DELETE FROM v11_test_chunks;" 2>/dev/null || true + sqlite3 exploration.db "DELETE FROM v11_test_strategies;" 2>/dev/null || true + echo "✓ Database cleared" +fi + +# Launch coordinator in background +echo "" +echo "🚀 Starting progressive sweep coordinator..." +nohup python3 v11_test_coordinator.py > coordinator_v11_progressive.log 2>&1 & +COORDINATOR_PID=$! + +echo "✓ Coordinator started (PID: $COORDINATOR_PID)" +echo "" +echo "================================================================" +echo "MONITORING" +echo "================================================================" +echo "Live log: tail -f coordinator_v11_progressive.log" +echo "Database: sqlite3 exploration.db" +echo "Results: cluster/v11_test_results/*.csv" +echo "" +echo "Check status:" +echo " sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\"" +echo "" +echo "Analyze signal distribution by ADX:" +echo " sqlite3 exploration.db \\" +echo " \"SELECT json_extract(params, '$.adx_min') as adx_min, \\" +echo " AVG(total_trades) as avg_signals, COUNT(*) as configs \\" +echo " FROM v11_test_strategies \\" +echo " GROUP BY adx_min ORDER BY adx_min;\"" +echo "" +echo "To stop sweep:" +echo " kill $COORDINATOR_PID" +echo "================================================================" diff --git a/cluster/v11_test_coordinator.py b/cluster/v11_test_coordinator.py index dce3c55..e9469b0 100755 --- a/cluster/v11_test_coordinator.py +++ b/cluster/v11_test_coordinator.py @@ -1,12 +1,34 @@ #!/usr/bin/env python3 """ -V11 Test Parameter Sweep Coordinator +V11 PROGRESSIVE Parameter Sweep Coordinator -Coordinates 256-combination test sweep across 2 workers with smart scheduling. +STAGE 1: ULTRA-PERMISSIVE (Start from 0 filters) +Strategy: "Go upwards from 0 until you find something" + +Coordinates 256-combination progressive test sweep across 2 workers with smart scheduling. Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK). -Test sweep: 2 chunks × 128 combinations = 256 total -Expected runtime: 6-25 minutes depending on worker availability +Progressive grid (512 combinations = 2×4×2×2×2×2×2×2): +- flip_threshold: 0.4, 0.5 +- adx_min: 0, 5, 10, 15 (0 = disabled) +- long_pos_max: 95, 100 (very loose) +- short_pos_min: 0, 5 (0 = disabled) +- vol_min: 0.0, 0.5 (0 = disabled) +- entry_buffer_atr: 0.0, 0.10 (0 = disabled) +- rsi_long_min: 25, 30 (permissive) +- rsi_short_max: 75, 80 (permissive) + +Expected outcomes: +- adx_min=0 configs: 150-300 signals (almost no filtering) +- adx_min=5 configs: 80-150 signals (light filtering) +- adx_min=10 configs: 40-80 signals (moderate filtering) +- adx_min=15 configs: 10-40 signals (strict filtering) + +If all still 0 signals with adx_min=0: +→ Base Money Line calculation is broken (not the filters) + +Test sweep: 2 chunks × 256 combinations = 512 total +Expected runtime: 12-35 minutes depending on worker availability """ import sqlite3 @@ -39,7 +61,7 @@ WORKERS = { DATA_FILE = 'data/solusdt_5m.csv' DB_PATH = 'exploration.db' -CHUNK_SIZE = 128 # Each chunk processes 128 combinations +CHUNK_SIZE = 256 # Each chunk processes 256 combinations # Telegram configuration TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY' @@ -134,10 +156,10 @@ def init_database(): ) """) - # Register 2 chunks (256 combinations total) + # Register 2 chunks (512 combinations total) chunks = [ - ('v11_test_chunk_0000', 0, 128, 128), - ('v11_test_chunk_0001', 128, 256, 128), + ('v11_test_chunk_0000', 0, 256, 256), + ('v11_test_chunk_0001', 256, 512, 256), ] for chunk_id, start, end, total in chunks: @@ -325,12 +347,17 @@ def main(): signal.signal(signal.SIGTERM, signal_handler) print("\n" + "="*60) - print("V11 TEST PARAMETER SWEEP COORDINATOR") + print("V11 PROGRESSIVE PARAMETER SWEEP COORDINATOR") + print("Stage 1: Ultra-Permissive (start from 0 filters)") print("="*60) - print(f"Total combinations: 256 (2^8)") - print(f"Chunks: 2 × 128 combinations") + print(f"Total combinations: 512 (2×4×2×2×2×2×2×2)") + print(f"Chunks: 2 × 256 combinations") print(f"Workers: 2 × 27 cores (85% CPU)") - print(f"Expected runtime: 6-25 minutes") + print(f"Expected runtime: 12-35 minutes") + print("") + print("Progressive strategy: Start filters at 0 (disabled)") + print("Expected: adx_min=0 → 150-300 signals") + print(" adx_min=15 → 10-40 signals") print("="*60 + "\n") # Initialize database @@ -340,9 +367,10 @@ def main(): # Send start notification available_workers = get_available_workers() start_msg = ( - f"🚀 V11 Test Sweep STARTED\n\n" - f"Combinations: 256 (2^8)\n" - f"Chunks: 2 × 128 combos\n" + f"🚀 V11 Progressive Sweep STARTED\n" + f"Stage 1: Ultra-Permissive (start from 0)\n\n" + f"Combinations: 512 (2×4×2×2×2×2×2×2)\n" + f"Chunks: 2 × 256 combos\n" f"Workers: {len(available_workers)} available\n" f"- Worker 1: Always on (27 cores)\n" ) @@ -350,7 +378,8 @@ def main(): start_msg += f"- Worker 2: Active (27 cores)\n" else: start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n" - start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + start_msg += f"\nExpected: adx_min=0 → 150-300 signals\n" + start_msg += f"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" send_telegram_message(start_msg) @@ -412,20 +441,27 @@ def main(): duration_min = duration / 60 print("\n" + "="*60) - print("V11 TEST SWEEP COMPLETE!") + print("V11 PROGRESSIVE SWEEP COMPLETE!") print("="*60) print(f"Duration: {duration_min:.1f} minutes") print(f"Chunks: 2/2 completed") - print(f"Strategies: 256 tested") + print(f"Strategies: 512 tested") + print("") + print("Next: Analyze signal distribution by ADX threshold") + print(" sqlite3 exploration.db \"SELECT json_extract(params, '$.adx_min') as adx_min,") + print(" AVG(total_trades) as avg_signals, COUNT(*) as configs") + print(" FROM v11_test_strategies GROUP BY adx_min ORDER BY adx_min;\"") print("="*60 + "\n") # Send completion notification complete_msg = ( - f"✅ V11 Test Sweep COMPLETE\n\n" + f"✅ V11 Progressive Sweep COMPLETE\n\n" f"Duration: {duration_min:.1f} minutes\n" f"Chunks: 2/2 completed\n" - f"Strategies: 256 tested\n\n" - f"Check results:\n" + f"Strategies: 512 tested\n\n" + f"Next step: Analyze signal distribution\n" + f"Check if adx_min=0 configs generated signals\n\n" + f"Results location:\n" f"- cluster/v11_test_results/\n" f"- sqlite3 exploration.db\n\n" f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" diff --git a/cluster/v11_test_worker.py b/cluster/v11_test_worker.py index 57f95ed..ac682a2 100755 --- a/cluster/v11_test_worker.py +++ b/cluster/v11_test_worker.py @@ -2,18 +2,26 @@ """ V11 Test Parameter Sweep Worker -Processes chunks of v11 test parameter configurations (256 combinations total). +Processes chunks of v11 test parameter configurations (512 combinations total). Uses 27 cores (85% CPU) for multiprocessing. -Test parameter grid (2 values each = 2^8 = 256 combinations): -- flip_threshold: 0.5, 0.6 -- adx_min: 18, 21 -- long_pos_max: 75, 80 -- short_pos_min: 20, 25 -- vol_min: 0.8, 1.0 -- entry_buffer_atr: 0.15, 0.20 -- rsi_long_min: 35, 40 -- rsi_short_max: 65, 70 +PROGRESSIVE SWEEP - Stage 1: Ultra-Permissive (start from 0 filters) +Goal: Find which parameter values allow signals through. + +Test parameter grid (2×4×2×2×2×2×2×2 = 512 combinations): +- flip_threshold: 0.4, 0.5 +- adx_min: 0, 5, 10, 15 (START FROM ZERO - filter disabled at 0) +- long_pos_max: 95, 100 (very loose) +- short_pos_min: 0, 5 (START FROM ZERO - filter disabled at 0) +- vol_min: 0.0, 0.5 (START FROM ZERO - filter disabled at 0) +- entry_buffer_atr: 0.0, 0.10 (START FROM ZERO - filter disabled at 0) +- rsi_long_min: 25, 30 (permissive) +- rsi_short_max: 75, 80 (permissive) + +Expected outcomes: +- adx_min=0 configs: 150-300 signals (almost no filtering) +- adx_min=15 configs: 10-40 signals (strict filtering) +- If all still 0 → base indicator broken, not the filters """ import sys @@ -45,17 +53,22 @@ def init_worker(data_file): global _DATA_FILE _DATA_FILE = data_file -# Test parameter grid (256 combinations) +# PROGRESSIVE Test parameter grid (512 combinations) +# Stage 1: Ultra-permissive - Start from 0 (filters disabled) to find baseline +# Strategy: "Go upwards from 0 until you find something" PARAMETER_GRID = { - 'flip_threshold': [0.5, 0.6], - 'adx_min': [18, 21], - 'long_pos_max': [75, 80], - 'short_pos_min': [20, 25], - 'vol_min': [0.8, 1.0], - 'entry_buffer_atr': [0.15, 0.20], - 'rsi_long_min': [35, 40], - 'rsi_short_max': [65, 70], + 'flip_threshold': [0.4, 0.5], # 2 values - range: loose to normal + 'adx_min': [0, 5, 10, 15], # 4 values - START FROM 0 (no filter) + 'long_pos_max': [95, 100], # 2 values - very permissive + 'short_pos_min': [0, 5], # 2 values - START FROM 0 (no filter) + 'vol_min': [0.0, 0.5], # 2 values - START FROM 0 (no filter) + 'entry_buffer_atr': [0.0, 0.10], # 2 values - START FROM 0 (no filter) + 'rsi_long_min': [25, 30], # 2 values - permissive + 'rsi_short_max': [75, 80], # 2 values - permissive } +# Total: 2×4×2×2×2×2×2×2 = 512 combos +# Expected: adx_min=0 configs will generate 150-300 signals (proves v11 logic works) +# If all still 0 signals with adx_min=0 → base indicator broken, not the filters def load_market_data(csv_file: str) -> pd.DataFrame: @@ -309,7 +322,7 @@ if __name__ == '__main__': chunk_id = sys.argv[2] start_idx = int(sys.argv[3]) - # Calculate end index (128 combos per chunk) - end_idx = start_idx + 128 + # Calculate end index (256 combos per chunk) + end_idx = start_idx + 256 process_chunk(data_file, chunk_id, start_idx, end_idx)