feat: Implement v11 progressive parameter sweep starting from zero filters

Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2025-12-06 20:30:57 +00:00
parent e92ba6df83
commit f678a027c2
4 changed files with 233 additions and 70 deletions

View File

@@ -5,19 +5,23 @@ CRITICAL DIFFERENCE FROM v9:
- v11: ALL filters actually applied to signals (useQualityFilters toggle)
- v9 bug: Filters calculated but signals ignored them
PROGRESSIVE SWEEP SUPPORT:
- Filter value of 0 = filter disabled (pass everything)
- Allows testing from ultra-permissive to strict restrictions
Based on moneyline_v11_all_filters.pinescript lines 271-272:
finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk))
finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk))
Test sweep parameters (8 params × 2 values = 256 combinations):
- flip_threshold: 0.5, 0.6
- adx_min: 18, 21
- long_pos_max: 75, 80
- short_pos_min: 20, 25
- vol_min: 0.8, 1.0
- entry_buffer_atr: 0.15, 0.20
- rsi_long_min: 35, 40
- rsi_short_max: 65, 70
Progressive test sweep parameters (512 combinations):
- flip_threshold: 0.4, 0.5
- adx_min: 0, 5, 10, 15 (0 = disabled)
- long_pos_max: 95, 100
- short_pos_min: 0, 5 (0 = disabled)
- vol_min: 0.0, 0.5 (0 = disabled)
- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
- rsi_long_min: 25, 30
- rsi_short_max: 75, 80
"""
from __future__ import annotations
@@ -39,7 +43,7 @@ Direction = Literal["long", "short"]
@dataclass
class MoneyLineV11Inputs:
"""v11 Money Line indicator parameters for test sweep."""
"""v11 Money Line indicator parameters for progressive test sweep."""
# Basic Money Line parameters (fixed for test)
confirm_bars: int = 0 # Immediate signals
@@ -49,15 +53,16 @@ class MoneyLineV11Inputs:
atr_period: int = 12 # ATR calculation length
multiplier: float = 3.8 # ATR band multiplier
# Filter parameters (8 parameters being optimized)
flip_threshold: float = 0.5 # % price must move to flip (TEST: 0.5, 0.6)
adx_min: float = 21 # Minimum ADX for signal (TEST: 18, 21)
long_pos_max: float = 75 # Don't long above X% of range (TEST: 75, 80)
short_pos_min: float = 20 # Don't short below X% of range (TEST: 20, 25)
vol_min: float = 1.0 # Minimum volume ratio (TEST: 0.8, 1.0)
entry_buffer_atr: float = 0.20 # ATR buffer beyond line (TEST: 0.15, 0.20)
rsi_long_min: float = 35 # RSI minimum for longs (TEST: 35, 40)
rsi_short_max: float = 70 # RSI maximum for shorts (TEST: 65, 70)
# PROGRESSIVE Filter parameters (256 combinations)
# Value of 0 = filter disabled (pass everything)
flip_threshold: float = 0.5 # % price must move to flip (PROGRESSIVE: 0.4, 0.5)
adx_min: float = 15 # Minimum ADX for signal (PROGRESSIVE: 0, 5, 10, 15 - 0=disabled)
long_pos_max: float = 95 # Don't long above X% of range (PROGRESSIVE: 95, 100)
short_pos_min: float = 0 # Don't short below X% of range (PROGRESSIVE: 0, 5 - 0=disabled)
vol_min: float = 0.5 # Minimum volume ratio (PROGRESSIVE: 0.0, 0.5 - 0=disabled)
entry_buffer_atr: float = 0.10 # ATR buffer beyond line (PROGRESSIVE: 0.0, 0.10 - 0=disabled)
rsi_long_min: float = 25 # RSI minimum for longs (PROGRESSIVE: 25, 30)
rsi_short_max: float = 80 # RSI maximum for shorts (PROGRESSIVE: 75, 80)
# Fixed filter parameters (not being optimized in test)
adx_length: int = 16 # ADX calculation length
@@ -259,16 +264,26 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
continue
# V11 CRITICAL: Apply ALL filters (this is what was broken in v9)
# PROGRESSIVE SWEEP: Value of 0 = filter disabled (pass everything)
# ADX filter (adxOk)
adx_ok = row.adx >= inputs.adx_min
# ADX filter (adxOk) - disabled if adx_min == 0
if inputs.adx_min > 0:
adx_ok = row.adx >= inputs.adx_min
else:
adx_ok = True # Filter disabled
# Volume filter (volumeOk)
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
# Volume filter (volumeOk) - disabled if vol_min == 0
if inputs.vol_min > 0:
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
else:
volume_ok = row.volume_ratio <= inputs.vol_max # Only check upper bound
if flip_long:
# Entry buffer check (longBufferOk)
entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
# Entry buffer check (longBufferOk) - disabled if entry_buffer_atr == 0
if inputs.entry_buffer_atr > 0:
entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
else:
entry_buffer_ok = True # Filter disabled
# Long filters
rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max # rsiLongOk
@@ -291,12 +306,20 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
cooldown_remaining = inputs.cooldown_bars
elif flip_short:
# Entry buffer check (shortBufferOk)
entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr)
# Entry buffer check (shortBufferOk) - disabled if entry_buffer_atr == 0
if inputs.entry_buffer_atr > 0:
entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr)
else:
entry_buffer_ok = True # Filter disabled
# Short filters
rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max # rsiShortOk
pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk
# Price position filter - disabled if short_pos_min == 0
if inputs.short_pos_min > 0:
pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk
else:
pos_ok = True # Filter disabled
# V11: ALL filters must pass (this is the fix from v9)
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:

View File

@@ -0,0 +1,91 @@
#!/bin/bash
# V11 PROGRESSIVE Parameter Sweep Launch Script
# Stage 1: Ultra-permissive (start from 0 filters) to find baseline
set -e # Exit on error
echo "================================================================"
echo "V11 PROGRESSIVE PARAMETER SWEEP - STAGE 1"
echo "================================================================"
echo ""
echo "Strategy: Start from 0 (filters disabled) and go upwards"
echo ""
echo "Progressive Grid (512 combinations):"
echo " - flip_threshold: 0.4, 0.5"
echo " - adx_min: 0, 5, 10, 15 (0 = disabled)"
echo " - long_pos_max: 95, 100 (very loose)"
echo " - short_pos_min: 0, 5 (0 = disabled)"
echo " - vol_min: 0.0, 0.5 (0 = disabled)"
echo " - entry_buffer_atr: 0.0, 0.10 (0 = disabled)"
echo " - rsi_long_min: 25, 30 (permissive)"
echo " - rsi_short_max: 75, 80 (permissive)"
echo ""
echo "Expected outcomes:"
echo " - adx_min=0 configs: 150-300 signals (almost no filtering)"
echo " - adx_min=5 configs: 80-150 signals (light filtering)"
echo " - adx_min=10 configs: 40-80 signals (moderate filtering)"
echo " - adx_min=15 configs: 10-40 signals (strict filtering)"
echo ""
echo "If all still 0 signals with adx_min=0:"
echo " → Base Money Line calculation is broken (not the filters)"
echo ""
echo "================================================================"
echo ""
cd "$(dirname "$0")"
# Check if data file exists
if [ ! -f "data/solusdt_5m.csv" ]; then
echo "✗ Error: data/solusdt_5m.csv not found"
echo " Please ensure market data is available"
exit 1
fi
echo "✓ Market data found"
# Check if coordinator script exists
if [ ! -f "v11_test_coordinator.py" ]; then
echo "✗ Error: v11_test_coordinator.py not found"
exit 1
fi
echo "✓ Coordinator script found"
# Clear previous test data
echo ""
echo "🗑️ Clearing previous test data..."
rm -rf v11_test_results/*
if [ -f "exploration.db" ]; then
sqlite3 exploration.db "DELETE FROM v11_test_chunks;" 2>/dev/null || true
sqlite3 exploration.db "DELETE FROM v11_test_strategies;" 2>/dev/null || true
echo "✓ Database cleared"
fi
# Launch coordinator in background
echo ""
echo "🚀 Starting progressive sweep coordinator..."
nohup python3 v11_test_coordinator.py > coordinator_v11_progressive.log 2>&1 &
COORDINATOR_PID=$!
echo "✓ Coordinator started (PID: $COORDINATOR_PID)"
echo ""
echo "================================================================"
echo "MONITORING"
echo "================================================================"
echo "Live log: tail -f coordinator_v11_progressive.log"
echo "Database: sqlite3 exploration.db"
echo "Results: cluster/v11_test_results/*.csv"
echo ""
echo "Check status:"
echo " sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\""
echo ""
echo "Analyze signal distribution by ADX:"
echo " sqlite3 exploration.db \\"
echo " \"SELECT json_extract(params, '$.adx_min') as adx_min, \\"
echo " AVG(total_trades) as avg_signals, COUNT(*) as configs \\"
echo " FROM v11_test_strategies \\"
echo " GROUP BY adx_min ORDER BY adx_min;\""
echo ""
echo "To stop sweep:"
echo " kill $COORDINATOR_PID"
echo "================================================================"

View File

@@ -1,12 +1,34 @@
#!/usr/bin/env python3
"""
V11 Test Parameter Sweep Coordinator
V11 PROGRESSIVE Parameter Sweep Coordinator
Coordinates 256-combination test sweep across 2 workers with smart scheduling.
STAGE 1: ULTRA-PERMISSIVE (Start from 0 filters)
Strategy: "Go upwards from 0 until you find something"
Coordinates 256-combination progressive test sweep across 2 workers with smart scheduling.
Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK).
Test sweep: 2 chunks × 128 combinations = 256 total
Expected runtime: 6-25 minutes depending on worker availability
Progressive grid (512 combinations = 2×4×2×2×2×2×2×2):
- flip_threshold: 0.4, 0.5
- adx_min: 0, 5, 10, 15 (0 = disabled)
- long_pos_max: 95, 100 (very loose)
- short_pos_min: 0, 5 (0 = disabled)
- vol_min: 0.0, 0.5 (0 = disabled)
- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
- rsi_long_min: 25, 30 (permissive)
- rsi_short_max: 75, 80 (permissive)
Expected outcomes:
- adx_min=0 configs: 150-300 signals (almost no filtering)
- adx_min=5 configs: 80-150 signals (light filtering)
- adx_min=10 configs: 40-80 signals (moderate filtering)
- adx_min=15 configs: 10-40 signals (strict filtering)
If all still 0 signals with adx_min=0:
→ Base Money Line calculation is broken (not the filters)
Test sweep: 2 chunks × 256 combinations = 512 total
Expected runtime: 12-35 minutes depending on worker availability
"""
import sqlite3
@@ -39,7 +61,7 @@ WORKERS = {
DATA_FILE = 'data/solusdt_5m.csv'
DB_PATH = 'exploration.db'
CHUNK_SIZE = 128 # Each chunk processes 128 combinations
CHUNK_SIZE = 256 # Each chunk processes 256 combinations
# Telegram configuration
TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY'
@@ -134,10 +156,10 @@ def init_database():
)
""")
# Register 2 chunks (256 combinations total)
# Register 2 chunks (512 combinations total)
chunks = [
('v11_test_chunk_0000', 0, 128, 128),
('v11_test_chunk_0001', 128, 256, 128),
('v11_test_chunk_0000', 0, 256, 256),
('v11_test_chunk_0001', 256, 512, 256),
]
for chunk_id, start, end, total in chunks:
@@ -325,12 +347,17 @@ def main():
signal.signal(signal.SIGTERM, signal_handler)
print("\n" + "="*60)
print("V11 TEST PARAMETER SWEEP COORDINATOR")
print("V11 PROGRESSIVE PARAMETER SWEEP COORDINATOR")
print("Stage 1: Ultra-Permissive (start from 0 filters)")
print("="*60)
print(f"Total combinations: 256 (2^8)")
print(f"Chunks: 2 × 128 combinations")
print(f"Total combinations: 512 (2×4×2×2×2×2×2×2)")
print(f"Chunks: 2 × 256 combinations")
print(f"Workers: 2 × 27 cores (85% CPU)")
print(f"Expected runtime: 6-25 minutes")
print(f"Expected runtime: 12-35 minutes")
print("")
print("Progressive strategy: Start filters at 0 (disabled)")
print("Expected: adx_min=0 → 150-300 signals")
print(" adx_min=15 → 10-40 signals")
print("="*60 + "\n")
# Initialize database
@@ -340,9 +367,10 @@ def main():
# Send start notification
available_workers = get_available_workers()
start_msg = (
f"🚀 <b>V11 Test Sweep STARTED</b>\n\n"
f"Combinations: 256 (2^8)\n"
f"Chunks: 2 × 128 combos\n"
f"🚀 <b>V11 Progressive Sweep STARTED</b>\n"
f"Stage 1: Ultra-Permissive (start from 0)\n\n"
f"Combinations: 512 (2×4×2×2×2×2×2×2)\n"
f"Chunks: 2 × 256 combos\n"
f"Workers: {len(available_workers)} available\n"
f"- Worker 1: Always on (27 cores)\n"
)
@@ -350,7 +378,8 @@ def main():
start_msg += f"- Worker 2: Active (27 cores)\n"
else:
start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n"
start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
start_msg += f"\nExpected: adx_min=0 → 150-300 signals\n"
start_msg += f"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
send_telegram_message(start_msg)
@@ -412,20 +441,27 @@ def main():
duration_min = duration / 60
print("\n" + "="*60)
print("V11 TEST SWEEP COMPLETE!")
print("V11 PROGRESSIVE SWEEP COMPLETE!")
print("="*60)
print(f"Duration: {duration_min:.1f} minutes")
print(f"Chunks: 2/2 completed")
print(f"Strategies: 256 tested")
print(f"Strategies: 512 tested")
print("")
print("Next: Analyze signal distribution by ADX threshold")
print(" sqlite3 exploration.db \"SELECT json_extract(params, '$.adx_min') as adx_min,")
print(" AVG(total_trades) as avg_signals, COUNT(*) as configs")
print(" FROM v11_test_strategies GROUP BY adx_min ORDER BY adx_min;\"")
print("="*60 + "\n")
# Send completion notification
complete_msg = (
f"✅ <b>V11 Test Sweep COMPLETE</b>\n\n"
f"✅ <b>V11 Progressive Sweep COMPLETE</b>\n\n"
f"Duration: {duration_min:.1f} minutes\n"
f"Chunks: 2/2 completed\n"
f"Strategies: 256 tested\n\n"
f"Check results:\n"
f"Strategies: 512 tested\n\n"
f"Next step: Analyze signal distribution\n"
f"Check if adx_min=0 configs generated signals\n\n"
f"Results location:\n"
f"- cluster/v11_test_results/\n"
f"- sqlite3 exploration.db\n\n"
f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"

View File

@@ -2,18 +2,26 @@
"""
V11 Test Parameter Sweep Worker
Processes chunks of v11 test parameter configurations (256 combinations total).
Processes chunks of v11 test parameter configurations (512 combinations total).
Uses 27 cores (85% CPU) for multiprocessing.
Test parameter grid (2 values each = 2^8 = 256 combinations):
- flip_threshold: 0.5, 0.6
- adx_min: 18, 21
- long_pos_max: 75, 80
- short_pos_min: 20, 25
- vol_min: 0.8, 1.0
- entry_buffer_atr: 0.15, 0.20
- rsi_long_min: 35, 40
- rsi_short_max: 65, 70
PROGRESSIVE SWEEP - Stage 1: Ultra-Permissive (start from 0 filters)
Goal: Find which parameter values allow signals through.
Test parameter grid (2×4×2×2×2×2×2×2 = 512 combinations):
- flip_threshold: 0.4, 0.5
- adx_min: 0, 5, 10, 15 (START FROM ZERO - filter disabled at 0)
- long_pos_max: 95, 100 (very loose)
- short_pos_min: 0, 5 (START FROM ZERO - filter disabled at 0)
- vol_min: 0.0, 0.5 (START FROM ZERO - filter disabled at 0)
- entry_buffer_atr: 0.0, 0.10 (START FROM ZERO - filter disabled at 0)
- rsi_long_min: 25, 30 (permissive)
- rsi_short_max: 75, 80 (permissive)
Expected outcomes:
- adx_min=0 configs: 150-300 signals (almost no filtering)
- adx_min=15 configs: 10-40 signals (strict filtering)
- If all still 0 → base indicator broken, not the filters
"""
import sys
@@ -45,17 +53,22 @@ def init_worker(data_file):
global _DATA_FILE
_DATA_FILE = data_file
# Test parameter grid (256 combinations)
# PROGRESSIVE Test parameter grid (512 combinations)
# Stage 1: Ultra-permissive - Start from 0 (filters disabled) to find baseline
# Strategy: "Go upwards from 0 until you find something"
PARAMETER_GRID = {
'flip_threshold': [0.5, 0.6],
'adx_min': [18, 21],
'long_pos_max': [75, 80],
'short_pos_min': [20, 25],
'vol_min': [0.8, 1.0],
'entry_buffer_atr': [0.15, 0.20],
'rsi_long_min': [35, 40],
'rsi_short_max': [65, 70],
'flip_threshold': [0.4, 0.5], # 2 values - range: loose to normal
'adx_min': [0, 5, 10, 15], # 4 values - START FROM 0 (no filter)
'long_pos_max': [95, 100], # 2 values - very permissive
'short_pos_min': [0, 5], # 2 values - START FROM 0 (no filter)
'vol_min': [0.0, 0.5], # 2 values - START FROM 0 (no filter)
'entry_buffer_atr': [0.0, 0.10], # 2 values - START FROM 0 (no filter)
'rsi_long_min': [25, 30], # 2 values - permissive
'rsi_short_max': [75, 80], # 2 values - permissive
}
# Total: 2×4×2×2×2×2×2×2 = 512 combos
# Expected: adx_min=0 configs will generate 150-300 signals (proves v11 logic works)
# If all still 0 signals with adx_min=0 → base indicator broken, not the filters
def load_market_data(csv_file: str) -> pd.DataFrame:
@@ -309,7 +322,7 @@ if __name__ == '__main__':
chunk_id = sys.argv[2]
start_idx = int(sys.argv[3])
# Calculate end index (128 combos per chunk)
end_idx = start_idx + 128
# Calculate end index (256 combos per chunk)
end_idx = start_idx + 256
process_chunk(data_file, chunk_id, start_idx, end_idx)