root/trading_bot_v4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement v11 progressive parameter sweep starting from zero filters

Browse Source 
Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2025-12-06 20:30:57 +00:00
parent e92ba6df83
commit f678a027c2
4 changed files with 233 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
backtester/v11_moneyline_all_filters.py
View File

@@ -5,19 +5,23 @@ CRITICAL DIFFERENCE FROM v9:
- v11: ALL filters actually applied to signals (useQualityFilters toggle) - v11: ALL filters actually applied to signals (useQualityFilters toggle)
- v9 bug: Filters calculated but signals ignored them - v9 bug: Filters calculated but signals ignored them
PROGRESSIVE SWEEP SUPPORT:
- Filter value of 0 = filter disabled (pass everything)
- Allows testing from ultra-permissive to strict restrictions
Based on moneyline_v11_all_filters.pinescript lines 271-272: Based on moneyline_v11_all_filters.pinescript lines 271-272:
finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk)) finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk))
finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk)) finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk))
Test sweep parameters (8 params × 2 values = 256 combinations): Progressive test sweep parameters (512 combinations):
- flip_threshold: 0.5, 0.6 - flip_threshold: 0.4, 0.5
- adx_min: 18, 21 - adx_min: 0, 5, 10, 15 (0 = disabled)
- long_pos_max: 75, 80 - long_pos_max: 95, 100
- short_pos_min: 20, 25 - short_pos_min: 0, 5 (0 = disabled)
- vol_min: 0.8, 1.0 - vol_min: 0.0, 0.5 (0 = disabled)
- entry_buffer_atr: 0.15, 0.20 - entry_buffer_atr: 0.0, 0.10 (0 = disabled)
- rsi_long_min: 35, 40 - rsi_long_min: 25, 30
- rsi_short_max: 65, 70 - rsi_short_max: 75, 80
""" """
from __future__ import annotations from __future__ import annotations
@@ -39,7 +43,7 @@ Direction = Literal["long", "short"]
@dataclass @dataclass
class MoneyLineV11Inputs: class MoneyLineV11Inputs:
"""v11 Money Line indicator parameters for test sweep.""" """v11 Money Line indicator parameters for progressive test sweep."""
# Basic Money Line parameters (fixed for test) # Basic Money Line parameters (fixed for test)
confirm_bars: int = 0 # Immediate signals confirm_bars: int = 0 # Immediate signals
@@ -49,15 +53,16 @@ class MoneyLineV11Inputs:
atr_period: int = 12 # ATR calculation length atr_period: int = 12 # ATR calculation length
multiplier: float = 3.8 # ATR band multiplier multiplier: float = 3.8 # ATR band multiplier
# Filter parameters (8 parameters being optimized) # PROGRESSIVE Filter parameters (256 combinations)
flip_threshold: float = 0.5 # % price must move to flip (TEST: 0.5, 0.6) # Value of 0 = filter disabled (pass everything)
adx_min: float = 21 # Minimum ADX for signal (TEST: 18, 21) flip_threshold: float = 0.5 # % price must move to flip (PROGRESSIVE: 0.4, 0.5)
long_pos_max: float = 75 # Don't long above X% of range (TEST: 75, 80) adx_min: float = 15 # Minimum ADX for signal (PROGRESSIVE: 0, 5, 10, 15 - 0=disabled)
short_pos_min: float = 20 # Don't short below X% of range (TEST: 20, 25) long_pos_max: float = 95 # Don't long above X% of range (PROGRESSIVE: 95, 100)
vol_min: float = 1.0 # Minimum volume ratio (TEST: 0.8, 1.0) short_pos_min: float = 0 # Don't short below X% of range (PROGRESSIVE: 0, 5 - 0=disabled)
entry_buffer_atr: float = 0.20 # ATR buffer beyond line (TEST: 0.15, 0.20) vol_min: float = 0.5 # Minimum volume ratio (PROGRESSIVE: 0.0, 0.5 - 0=disabled)
rsi_long_min: float = 35 # RSI minimum for longs (TEST: 35, 40) entry_buffer_atr: float = 0.10 # ATR buffer beyond line (PROGRESSIVE: 0.0, 0.10 - 0=disabled)
rsi_short_max: float = 70 # RSI maximum for shorts (TEST: 65, 70) rsi_long_min: float = 25 # RSI minimum for longs (PROGRESSIVE: 25, 30)
rsi_short_max: float = 80 # RSI maximum for shorts (PROGRESSIVE: 75, 80)
# Fixed filter parameters (not being optimized in test) # Fixed filter parameters (not being optimized in test)
adx_length: int = 16 # ADX calculation length adx_length: int = 16 # ADX calculation length
@@ -259,16 +264,26 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
continue continue
# V11 CRITICAL: Apply ALL filters (this is what was broken in v9) # V11 CRITICAL: Apply ALL filters (this is what was broken in v9)
# PROGRESSIVE SWEEP: Value of 0 = filter disabled (pass everything)
# ADX filter (adxOk) # ADX filter (adxOk) - disabled if adx_min == 0
if inputs.adx_min > 0:
adx_ok = row.adx >= inputs.adx_min adx_ok = row.adx >= inputs.adx_min
else:
adx_ok = True # Filter disabled
# Volume filter (volumeOk) # Volume filter (volumeOk) - disabled if vol_min == 0
if inputs.vol_min > 0:
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
else:
volume_ok = row.volume_ratio <= inputs.vol_max # Only check upper bound
if flip_long: if flip_long:
# Entry buffer check (longBufferOk) # Entry buffer check (longBufferOk) - disabled if entry_buffer_atr == 0
if inputs.entry_buffer_atr > 0:
entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr) entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
else:
entry_buffer_ok = True # Filter disabled
# Long filters # Long filters
rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max # rsiLongOk rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max # rsiLongOk
@@ -291,12 +306,20 @@ def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs
cooldown_remaining = inputs.cooldown_bars cooldown_remaining = inputs.cooldown_bars
elif flip_short: elif flip_short:
# Entry buffer check (shortBufferOk) # Entry buffer check (shortBufferOk) - disabled if entry_buffer_atr == 0
if inputs.entry_buffer_atr > 0:
entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr) entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr)
else:
entry_buffer_ok = True # Filter disabled
# Short filters # Short filters
rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max # rsiShortOk rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max # rsiShortOk
# Price position filter - disabled if short_pos_min == 0
if inputs.short_pos_min > 0:
pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk
else:
pos_ok = True # Filter disabled
# V11: ALL filters must pass (this is the fix from v9) # V11: ALL filters must pass (this is the fix from v9)
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok: if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:

91
cluster/run_v11_progressive_sweep.sh Executable file
View File

@@ -0,0 +1,91 @@
#!/bin/bash
# V11 PROGRESSIVE Parameter Sweep Launch Script
# Stage 1: Ultra-permissive (start from 0 filters) to find baseline
set -e # Exit on error
echo "================================================================"
echo "V11 PROGRESSIVE PARAMETER SWEEP - STAGE 1"
echo "================================================================"
echo ""
echo "Strategy: Start from 0 (filters disabled) and go upwards"
echo ""
echo "Progressive Grid (512 combinations):"
echo " - flip_threshold: 0.4, 0.5"
echo " - adx_min: 0, 5, 10, 15 (0 = disabled)"
echo " - long_pos_max: 95, 100 (very loose)"
echo " - short_pos_min: 0, 5 (0 = disabled)"
echo " - vol_min: 0.0, 0.5 (0 = disabled)"
echo " - entry_buffer_atr: 0.0, 0.10 (0 = disabled)"
echo " - rsi_long_min: 25, 30 (permissive)"
echo " - rsi_short_max: 75, 80 (permissive)"
echo ""
echo "Expected outcomes:"
echo " - adx_min=0 configs: 150-300 signals (almost no filtering)"
echo " - adx_min=5 configs: 80-150 signals (light filtering)"
echo " - adx_min=10 configs: 40-80 signals (moderate filtering)"
echo " - adx_min=15 configs: 10-40 signals (strict filtering)"
echo ""
echo "If all still 0 signals with adx_min=0:"
echo " → Base Money Line calculation is broken (not the filters)"
echo ""
echo "================================================================"
echo ""
cd "$(dirname "$0")"
# Check if data file exists
if [ ! -f "data/solusdt_5m.csv" ]; then
echo "✗ Error: data/solusdt_5m.csv not found"
echo " Please ensure market data is available"
exit 1
fi
echo "✓ Market data found"
# Check if coordinator script exists
if [ ! -f "v11_test_coordinator.py" ]; then
echo "✗ Error: v11_test_coordinator.py not found"
exit 1
fi
echo "✓ Coordinator script found"
# Clear previous test data
echo ""
echo "🗑️ Clearing previous test data..."
rm -rf v11_test_results/*
if [ -f "exploration.db" ]; then
sqlite3 exploration.db "DELETE FROM v11_test_chunks;" 2>/dev/null || true
sqlite3 exploration.db "DELETE FROM v11_test_strategies;" 2>/dev/null || true
echo "✓ Database cleared"
fi
# Launch coordinator in background
echo ""
echo "🚀 Starting progressive sweep coordinator..."
nohup python3 v11_test_coordinator.py > coordinator_v11_progressive.log 2>&1 &
COORDINATOR_PID=$!
echo "✓ Coordinator started (PID: $COORDINATOR_PID)"
echo ""
echo "================================================================"
echo "MONITORING"
echo "================================================================"
echo "Live log: tail -f coordinator_v11_progressive.log"
echo "Database: sqlite3 exploration.db"
echo "Results: cluster/v11_test_results/*.csv"
echo ""
echo "Check status:"
echo " sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\""
echo ""
echo "Analyze signal distribution by ADX:"
echo " sqlite3 exploration.db \\"
echo " \"SELECT json_extract(params, '$.adx_min') as adx_min, \\"
echo " AVG(total_trades) as avg_signals, COUNT(*) as configs \\"
echo " FROM v11_test_strategies \\"
echo " GROUP BY adx_min ORDER BY adx_min;\""
echo ""
echo "To stop sweep:"
echo " kill $COORDINATOR_PID"
echo "================================================================"

78
cluster/v11_test_coordinator.py
View File

@@ -1,12 +1,34 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
V11 Test Parameter Sweep Coordinator V11 PROGRESSIVE Parameter Sweep Coordinator
Coordinates 256-combination test sweep across 2 workers with smart scheduling. STAGE 1: ULTRA-PERMISSIVE (Start from 0 filters)
Strategy: "Go upwards from 0 until you find something"
Coordinates 256-combination progressive test sweep across 2 workers with smart scheduling.
Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK). Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK).
Test sweep: 2 chunks × 128 combinations = 256 total Progressive grid (512 combinations = 2×4×2×2×2×2×2×2):
Expected runtime: 6-25 minutes depending on worker availability - flip_threshold: 0.4, 0.5
- adx_min: 0, 5, 10, 15 (0 = disabled)
- long_pos_max: 95, 100 (very loose)
- short_pos_min: 0, 5 (0 = disabled)
- vol_min: 0.0, 0.5 (0 = disabled)
- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
- rsi_long_min: 25, 30 (permissive)
- rsi_short_max: 75, 80 (permissive)
Expected outcomes:
- adx_min=0 configs: 150-300 signals (almost no filtering)
- adx_min=5 configs: 80-150 signals (light filtering)
- adx_min=10 configs: 40-80 signals (moderate filtering)
- adx_min=15 configs: 10-40 signals (strict filtering)
If all still 0 signals with adx_min=0:
→ Base Money Line calculation is broken (not the filters)
Test sweep: 2 chunks × 256 combinations = 512 total
Expected runtime: 12-35 minutes depending on worker availability
""" """
import sqlite3 import sqlite3
@@ -39,7 +61,7 @@ WORKERS = {
DATA_FILE = 'data/solusdt_5m.csv' DATA_FILE = 'data/solusdt_5m.csv'
DB_PATH = 'exploration.db' DB_PATH = 'exploration.db'
CHUNK_SIZE = 128 # Each chunk processes 128 combinations CHUNK_SIZE = 256 # Each chunk processes 256 combinations
# Telegram configuration # Telegram configuration
TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY' TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY'
@@ -134,10 +156,10 @@ def init_database():
) )
""") """)
# Register 2 chunks (256 combinations total) # Register 2 chunks (512 combinations total)
chunks = [ chunks = [
('v11_test_chunk_0000', 0, 128, 128), ('v11_test_chunk_0000', 0, 256, 256),
('v11_test_chunk_0001', 128, 256, 128), ('v11_test_chunk_0001', 256, 512, 256),
] ]
for chunk_id, start, end, total in chunks: for chunk_id, start, end, total in chunks:
@@ -325,12 +347,17 @@ def main():
signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
print("\n" + "="*60) print("\n" + "="*60)
print("V11 TEST PARAMETER SWEEP COORDINATOR") print("V11 PROGRESSIVE PARAMETER SWEEP COORDINATOR")
print("Stage 1: Ultra-Permissive (start from 0 filters)")
print("="*60) print("="*60)
print(f"Total combinations: 256 (2^8)") print(f"Total combinations: 512 (2×4×2×2×2×2×2×2)")
print(f"Chunks: 2 × 128 combinations") print(f"Chunks: 2 × 256 combinations")
print(f"Workers: 2 × 27 cores (85% CPU)") print(f"Workers: 2 × 27 cores (85% CPU)")
print(f"Expected runtime: 6-25 minutes") print(f"Expected runtime: 12-35 minutes")
print("")
print("Progressive strategy: Start filters at 0 (disabled)")
print("Expected: adx_min=0 → 150-300 signals")
print(" adx_min=15 → 10-40 signals")
print("="*60 + "\n") print("="*60 + "\n")
# Initialize database # Initialize database
@@ -340,9 +367,10 @@ def main():
# Send start notification # Send start notification
available_workers = get_available_workers() available_workers = get_available_workers()
start_msg = ( start_msg = (
f"🚀 <b>V11 Test Sweep STARTED</b>\n\n" f"🚀 <b>V11 Progressive Sweep STARTED</b>\n"
f"Combinations: 256 (2^8)\n" f"Stage 1: Ultra-Permissive (start from 0)\n\n"
f"Chunks: 2 × 128 combos\n" f"Combinations: 512 (2×4×2×2×2×2×2×2)\n"
f"Chunks: 2 × 256 combos\n"
f"Workers: {len(available_workers)} available\n" f"Workers: {len(available_workers)} available\n"
f"- Worker 1: Always on (27 cores)\n" f"- Worker 1: Always on (27 cores)\n"
) )
@@ -350,7 +378,8 @@ def main():
start_msg += f"- Worker 2: Active (27 cores)\n" start_msg += f"- Worker 2: Active (27 cores)\n"
else: else:
start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n" start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n"
start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" start_msg += f"\nExpected: adx_min=0 → 150-300 signals\n"
start_msg += f"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
send_telegram_message(start_msg) send_telegram_message(start_msg)
@@ -412,20 +441,27 @@ def main():
duration_min = duration / 60 duration_min = duration / 60
print("\n" + "="*60) print("\n" + "="*60)
print("V11 TEST SWEEP COMPLETE!") print("V11 PROGRESSIVE SWEEP COMPLETE!")
print("="*60) print("="*60)
print(f"Duration: {duration_min:.1f} minutes") print(f"Duration: {duration_min:.1f} minutes")
print(f"Chunks: 2/2 completed") print(f"Chunks: 2/2 completed")
print(f"Strategies: 256 tested") print(f"Strategies: 512 tested")
print("")
print("Next: Analyze signal distribution by ADX threshold")
print(" sqlite3 exploration.db \"SELECT json_extract(params, '$.adx_min') as adx_min,")
print(" AVG(total_trades) as avg_signals, COUNT(*) as configs")
print(" FROM v11_test_strategies GROUP BY adx_min ORDER BY adx_min;\"")
print("="*60 + "\n") print("="*60 + "\n")
# Send completion notification # Send completion notification
complete_msg = ( complete_msg = (
f"✅ <b>V11 Test Sweep COMPLETE</b>\n\n" f"✅ <b>V11 Progressive Sweep COMPLETE</b>\n\n"
f"Duration: {duration_min:.1f} minutes\n" f"Duration: {duration_min:.1f} minutes\n"
f"Chunks: 2/2 completed\n" f"Chunks: 2/2 completed\n"
f"Strategies: 256 tested\n\n" f"Strategies: 512 tested\n\n"
f"Check results:\n" f"Next step: Analyze signal distribution\n"
f"Check if adx_min=0 configs generated signals\n\n"
f"Results location:\n"
f"- cluster/v11_test_results/\n" f"- cluster/v11_test_results/\n"
f"- sqlite3 exploration.db\n\n" f"- sqlite3 exploration.db\n\n"
f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"

55
cluster/v11_test_worker.py
View File

@@ -2,18 +2,26 @@
""" """
V11 Test Parameter Sweep Worker V11 Test Parameter Sweep Worker
Processes chunks of v11 test parameter configurations (256 combinations total). Processes chunks of v11 test parameter configurations (512 combinations total).
Uses 27 cores (85% CPU) for multiprocessing. Uses 27 cores (85% CPU) for multiprocessing.
Test parameter grid (2 values each = 2^8 = 256 combinations): PROGRESSIVE SWEEP - Stage 1: Ultra-Permissive (start from 0 filters)
- flip_threshold: 0.5, 0.6 Goal: Find which parameter values allow signals through.
- adx_min: 18, 21
- long_pos_max: 75, 80 Test parameter grid (2×4×2×2×2×2×2×2 = 512 combinations):
- short_pos_min: 20, 25 - flip_threshold: 0.4, 0.5
- vol_min: 0.8, 1.0 - adx_min: 0, 5, 10, 15 (START FROM ZERO - filter disabled at 0)
- entry_buffer_atr: 0.15, 0.20 - long_pos_max: 95, 100 (very loose)
- rsi_long_min: 35, 40 - short_pos_min: 0, 5 (START FROM ZERO - filter disabled at 0)
- rsi_short_max: 65, 70 - vol_min: 0.0, 0.5 (START FROM ZERO - filter disabled at 0)
- entry_buffer_atr: 0.0, 0.10 (START FROM ZERO - filter disabled at 0)
- rsi_long_min: 25, 30 (permissive)
- rsi_short_max: 75, 80 (permissive)
Expected outcomes:
- adx_min=0 configs: 150-300 signals (almost no filtering)
- adx_min=15 configs: 10-40 signals (strict filtering)
- If all still 0 → base indicator broken, not the filters
""" """
import sys import sys
@@ -45,17 +53,22 @@ def init_worker(data_file):
global _DATA_FILE global _DATA_FILE
_DATA_FILE = data_file _DATA_FILE = data_file
# Test parameter grid (256 combinations) # PROGRESSIVE Test parameter grid (512 combinations)
# Stage 1: Ultra-permissive - Start from 0 (filters disabled) to find baseline
# Strategy: "Go upwards from 0 until you find something"
PARAMETER_GRID = { PARAMETER_GRID = {
'flip_threshold': [0.5, 0.6], 'flip_threshold': [0.4, 0.5], # 2 values - range: loose to normal
'adx_min': [18, 21], 'adx_min': [0, 5, 10, 15], # 4 values - START FROM 0 (no filter)
'long_pos_max': [75, 80], 'long_pos_max': [95, 100], # 2 values - very permissive
'short_pos_min': [20, 25], 'short_pos_min': [0, 5], # 2 values - START FROM 0 (no filter)
'vol_min': [0.8, 1.0], 'vol_min': [0.0, 0.5], # 2 values - START FROM 0 (no filter)
'entry_buffer_atr': [0.15, 0.20], 'entry_buffer_atr': [0.0, 0.10], # 2 values - START FROM 0 (no filter)
'rsi_long_min': [35, 40], 'rsi_long_min': [25, 30], # 2 values - permissive
'rsi_short_max': [65, 70], 'rsi_short_max': [75, 80], # 2 values - permissive
} }
# Total: 2×4×2×2×2×2×2×2 = 512 combos
# Expected: adx_min=0 configs will generate 150-300 signals (proves v11 logic works)
# If all still 0 signals with adx_min=0 → base indicator broken, not the filters
def load_market_data(csv_file: str) -> pd.DataFrame: def load_market_data(csv_file: str) -> pd.DataFrame:
@@ -309,7 +322,7 @@ if __name__ == '__main__':
chunk_id = sys.argv[2] chunk_id = sys.argv[2]
start_idx = int(sys.argv[3]) start_idx = int(sys.argv[3])
# Calculate end index (128 combos per chunk) # Calculate end index (256 combos per chunk)
end_idx = start_idx + 128 end_idx = start_idx + 256
process_chunk(data_file, chunk_id, start_idx, end_idx) process_chunk(data_file, chunk_id, start_idx, end_idx)