feat: Implement v11 progressive parameter sweep starting from zero filters
Co-authored-by: mindesbunister <32161838+mindesbunister@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
91
cluster/run_v11_progressive_sweep.sh
Executable file
91
cluster/run_v11_progressive_sweep.sh
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# V11 PROGRESSIVE Parameter Sweep Launch Script
|
||||
# Stage 1: Ultra-permissive (start from 0 filters) to find baseline
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "================================================================"
|
||||
echo "V11 PROGRESSIVE PARAMETER SWEEP - STAGE 1"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
echo "Strategy: Start from 0 (filters disabled) and go upwards"
|
||||
echo ""
|
||||
echo "Progressive Grid (512 combinations):"
|
||||
echo " - flip_threshold: 0.4, 0.5"
|
||||
echo " - adx_min: 0, 5, 10, 15 (0 = disabled)"
|
||||
echo " - long_pos_max: 95, 100 (very loose)"
|
||||
echo " - short_pos_min: 0, 5 (0 = disabled)"
|
||||
echo " - vol_min: 0.0, 0.5 (0 = disabled)"
|
||||
echo " - entry_buffer_atr: 0.0, 0.10 (0 = disabled)"
|
||||
echo " - rsi_long_min: 25, 30 (permissive)"
|
||||
echo " - rsi_short_max: 75, 80 (permissive)"
|
||||
echo ""
|
||||
echo "Expected outcomes:"
|
||||
echo " - adx_min=0 configs: 150-300 signals (almost no filtering)"
|
||||
echo " - adx_min=5 configs: 80-150 signals (light filtering)"
|
||||
echo " - adx_min=10 configs: 40-80 signals (moderate filtering)"
|
||||
echo " - adx_min=15 configs: 10-40 signals (strict filtering)"
|
||||
echo ""
|
||||
echo "If all still 0 signals with adx_min=0:"
|
||||
echo " → Base Money Line calculation is broken (not the filters)"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# Check if data file exists
|
||||
if [ ! -f "data/solusdt_5m.csv" ]; then
|
||||
echo "✗ Error: data/solusdt_5m.csv not found"
|
||||
echo " Please ensure market data is available"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Market data found"
|
||||
|
||||
# Check if coordinator script exists
|
||||
if [ ! -f "v11_test_coordinator.py" ]; then
|
||||
echo "✗ Error: v11_test_coordinator.py not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Coordinator script found"
|
||||
|
||||
# Clear previous test data
|
||||
echo ""
|
||||
echo "🗑️ Clearing previous test data..."
|
||||
rm -rf v11_test_results/*
|
||||
if [ -f "exploration.db" ]; then
|
||||
sqlite3 exploration.db "DELETE FROM v11_test_chunks;" 2>/dev/null || true
|
||||
sqlite3 exploration.db "DELETE FROM v11_test_strategies;" 2>/dev/null || true
|
||||
echo "✓ Database cleared"
|
||||
fi
|
||||
|
||||
# Launch coordinator in background
|
||||
echo ""
|
||||
echo "🚀 Starting progressive sweep coordinator..."
|
||||
nohup python3 v11_test_coordinator.py > coordinator_v11_progressive.log 2>&1 &
|
||||
COORDINATOR_PID=$!
|
||||
|
||||
echo "✓ Coordinator started (PID: $COORDINATOR_PID)"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "MONITORING"
|
||||
echo "================================================================"
|
||||
echo "Live log: tail -f coordinator_v11_progressive.log"
|
||||
echo "Database: sqlite3 exploration.db"
|
||||
echo "Results: cluster/v11_test_results/*.csv"
|
||||
echo ""
|
||||
echo "Check status:"
|
||||
echo " sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\""
|
||||
echo ""
|
||||
echo "Analyze signal distribution by ADX:"
|
||||
echo " sqlite3 exploration.db \\"
|
||||
echo " \"SELECT json_extract(params, '$.adx_min') as adx_min, \\"
|
||||
echo " AVG(total_trades) as avg_signals, COUNT(*) as configs \\"
|
||||
echo " FROM v11_test_strategies \\"
|
||||
echo " GROUP BY adx_min ORDER BY adx_min;\""
|
||||
echo ""
|
||||
echo "To stop sweep:"
|
||||
echo " kill $COORDINATOR_PID"
|
||||
echo "================================================================"
|
||||
@@ -1,12 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
V11 Test Parameter Sweep Coordinator
|
||||
V11 PROGRESSIVE Parameter Sweep Coordinator
|
||||
|
||||
Coordinates 256-combination test sweep across 2 workers with smart scheduling.
|
||||
STAGE 1: ULTRA-PERMISSIVE (Start from 0 filters)
|
||||
Strategy: "Go upwards from 0 until you find something"
|
||||
|
||||
Coordinates 256-combination progressive test sweep across 2 workers with smart scheduling.
|
||||
Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK).
|
||||
|
||||
Test sweep: 2 chunks × 128 combinations = 256 total
|
||||
Expected runtime: 6-25 minutes depending on worker availability
|
||||
Progressive grid (512 combinations = 2×4×2×2×2×2×2×2):
|
||||
- flip_threshold: 0.4, 0.5
|
||||
- adx_min: 0, 5, 10, 15 (0 = disabled)
|
||||
- long_pos_max: 95, 100 (very loose)
|
||||
- short_pos_min: 0, 5 (0 = disabled)
|
||||
- vol_min: 0.0, 0.5 (0 = disabled)
|
||||
- entry_buffer_atr: 0.0, 0.10 (0 = disabled)
|
||||
- rsi_long_min: 25, 30 (permissive)
|
||||
- rsi_short_max: 75, 80 (permissive)
|
||||
|
||||
Expected outcomes:
|
||||
- adx_min=0 configs: 150-300 signals (almost no filtering)
|
||||
- adx_min=5 configs: 80-150 signals (light filtering)
|
||||
- adx_min=10 configs: 40-80 signals (moderate filtering)
|
||||
- adx_min=15 configs: 10-40 signals (strict filtering)
|
||||
|
||||
If all still 0 signals with adx_min=0:
|
||||
→ Base Money Line calculation is broken (not the filters)
|
||||
|
||||
Test sweep: 2 chunks × 256 combinations = 512 total
|
||||
Expected runtime: 12-35 minutes depending on worker availability
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
@@ -39,7 +61,7 @@ WORKERS = {
|
||||
|
||||
DATA_FILE = 'data/solusdt_5m.csv'
|
||||
DB_PATH = 'exploration.db'
|
||||
CHUNK_SIZE = 128 # Each chunk processes 128 combinations
|
||||
CHUNK_SIZE = 256 # Each chunk processes 256 combinations
|
||||
|
||||
# Telegram configuration
|
||||
TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY'
|
||||
@@ -134,10 +156,10 @@ def init_database():
|
||||
)
|
||||
""")
|
||||
|
||||
# Register 2 chunks (256 combinations total)
|
||||
# Register 2 chunks (512 combinations total)
|
||||
chunks = [
|
||||
('v11_test_chunk_0000', 0, 128, 128),
|
||||
('v11_test_chunk_0001', 128, 256, 128),
|
||||
('v11_test_chunk_0000', 0, 256, 256),
|
||||
('v11_test_chunk_0001', 256, 512, 256),
|
||||
]
|
||||
|
||||
for chunk_id, start, end, total in chunks:
|
||||
@@ -325,12 +347,17 @@ def main():
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("V11 TEST PARAMETER SWEEP COORDINATOR")
|
||||
print("V11 PROGRESSIVE PARAMETER SWEEP COORDINATOR")
|
||||
print("Stage 1: Ultra-Permissive (start from 0 filters)")
|
||||
print("="*60)
|
||||
print(f"Total combinations: 256 (2^8)")
|
||||
print(f"Chunks: 2 × 128 combinations")
|
||||
print(f"Total combinations: 512 (2×4×2×2×2×2×2×2)")
|
||||
print(f"Chunks: 2 × 256 combinations")
|
||||
print(f"Workers: 2 × 27 cores (85% CPU)")
|
||||
print(f"Expected runtime: 6-25 minutes")
|
||||
print(f"Expected runtime: 12-35 minutes")
|
||||
print("")
|
||||
print("Progressive strategy: Start filters at 0 (disabled)")
|
||||
print("Expected: adx_min=0 → 150-300 signals")
|
||||
print(" adx_min=15 → 10-40 signals")
|
||||
print("="*60 + "\n")
|
||||
|
||||
# Initialize database
|
||||
@@ -340,9 +367,10 @@ def main():
|
||||
# Send start notification
|
||||
available_workers = get_available_workers()
|
||||
start_msg = (
|
||||
f"🚀 <b>V11 Test Sweep STARTED</b>\n\n"
|
||||
f"Combinations: 256 (2^8)\n"
|
||||
f"Chunks: 2 × 128 combos\n"
|
||||
f"🚀 <b>V11 Progressive Sweep STARTED</b>\n"
|
||||
f"Stage 1: Ultra-Permissive (start from 0)\n\n"
|
||||
f"Combinations: 512 (2×4×2×2×2×2×2×2)\n"
|
||||
f"Chunks: 2 × 256 combos\n"
|
||||
f"Workers: {len(available_workers)} available\n"
|
||||
f"- Worker 1: Always on (27 cores)\n"
|
||||
)
|
||||
@@ -350,7 +378,8 @@ def main():
|
||||
start_msg += f"- Worker 2: Active (27 cores)\n"
|
||||
else:
|
||||
start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n"
|
||||
start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
start_msg += f"\nExpected: adx_min=0 → 150-300 signals\n"
|
||||
start_msg += f"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
|
||||
send_telegram_message(start_msg)
|
||||
|
||||
@@ -412,20 +441,27 @@ def main():
|
||||
duration_min = duration / 60
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("V11 TEST SWEEP COMPLETE!")
|
||||
print("V11 PROGRESSIVE SWEEP COMPLETE!")
|
||||
print("="*60)
|
||||
print(f"Duration: {duration_min:.1f} minutes")
|
||||
print(f"Chunks: 2/2 completed")
|
||||
print(f"Strategies: 256 tested")
|
||||
print(f"Strategies: 512 tested")
|
||||
print("")
|
||||
print("Next: Analyze signal distribution by ADX threshold")
|
||||
print(" sqlite3 exploration.db \"SELECT json_extract(params, '$.adx_min') as adx_min,")
|
||||
print(" AVG(total_trades) as avg_signals, COUNT(*) as configs")
|
||||
print(" FROM v11_test_strategies GROUP BY adx_min ORDER BY adx_min;\"")
|
||||
print("="*60 + "\n")
|
||||
|
||||
# Send completion notification
|
||||
complete_msg = (
|
||||
f"✅ <b>V11 Test Sweep COMPLETE</b>\n\n"
|
||||
f"✅ <b>V11 Progressive Sweep COMPLETE</b>\n\n"
|
||||
f"Duration: {duration_min:.1f} minutes\n"
|
||||
f"Chunks: 2/2 completed\n"
|
||||
f"Strategies: 256 tested\n\n"
|
||||
f"Check results:\n"
|
||||
f"Strategies: 512 tested\n\n"
|
||||
f"Next step: Analyze signal distribution\n"
|
||||
f"Check if adx_min=0 configs generated signals\n\n"
|
||||
f"Results location:\n"
|
||||
f"- cluster/v11_test_results/\n"
|
||||
f"- sqlite3 exploration.db\n\n"
|
||||
f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
|
||||
@@ -2,18 +2,26 @@
|
||||
"""
|
||||
V11 Test Parameter Sweep Worker
|
||||
|
||||
Processes chunks of v11 test parameter configurations (256 combinations total).
|
||||
Processes chunks of v11 test parameter configurations (512 combinations total).
|
||||
Uses 27 cores (85% CPU) for multiprocessing.
|
||||
|
||||
Test parameter grid (2 values each = 2^8 = 256 combinations):
|
||||
- flip_threshold: 0.5, 0.6
|
||||
- adx_min: 18, 21
|
||||
- long_pos_max: 75, 80
|
||||
- short_pos_min: 20, 25
|
||||
- vol_min: 0.8, 1.0
|
||||
- entry_buffer_atr: 0.15, 0.20
|
||||
- rsi_long_min: 35, 40
|
||||
- rsi_short_max: 65, 70
|
||||
PROGRESSIVE SWEEP - Stage 1: Ultra-Permissive (start from 0 filters)
|
||||
Goal: Find which parameter values allow signals through.
|
||||
|
||||
Test parameter grid (2×4×2×2×2×2×2×2 = 512 combinations):
|
||||
- flip_threshold: 0.4, 0.5
|
||||
- adx_min: 0, 5, 10, 15 (START FROM ZERO - filter disabled at 0)
|
||||
- long_pos_max: 95, 100 (very loose)
|
||||
- short_pos_min: 0, 5 (START FROM ZERO - filter disabled at 0)
|
||||
- vol_min: 0.0, 0.5 (START FROM ZERO - filter disabled at 0)
|
||||
- entry_buffer_atr: 0.0, 0.10 (START FROM ZERO - filter disabled at 0)
|
||||
- rsi_long_min: 25, 30 (permissive)
|
||||
- rsi_short_max: 75, 80 (permissive)
|
||||
|
||||
Expected outcomes:
|
||||
- adx_min=0 configs: 150-300 signals (almost no filtering)
|
||||
- adx_min=15 configs: 10-40 signals (strict filtering)
|
||||
- If all still 0 → base indicator broken, not the filters
|
||||
"""
|
||||
|
||||
import sys
|
||||
@@ -45,17 +53,22 @@ def init_worker(data_file):
|
||||
global _DATA_FILE
|
||||
_DATA_FILE = data_file
|
||||
|
||||
# Test parameter grid (256 combinations)
|
||||
# PROGRESSIVE Test parameter grid (512 combinations)
|
||||
# Stage 1: Ultra-permissive - Start from 0 (filters disabled) to find baseline
|
||||
# Strategy: "Go upwards from 0 until you find something"
|
||||
PARAMETER_GRID = {
|
||||
'flip_threshold': [0.5, 0.6],
|
||||
'adx_min': [18, 21],
|
||||
'long_pos_max': [75, 80],
|
||||
'short_pos_min': [20, 25],
|
||||
'vol_min': [0.8, 1.0],
|
||||
'entry_buffer_atr': [0.15, 0.20],
|
||||
'rsi_long_min': [35, 40],
|
||||
'rsi_short_max': [65, 70],
|
||||
'flip_threshold': [0.4, 0.5], # 2 values - range: loose to normal
|
||||
'adx_min': [0, 5, 10, 15], # 4 values - START FROM 0 (no filter)
|
||||
'long_pos_max': [95, 100], # 2 values - very permissive
|
||||
'short_pos_min': [0, 5], # 2 values - START FROM 0 (no filter)
|
||||
'vol_min': [0.0, 0.5], # 2 values - START FROM 0 (no filter)
|
||||
'entry_buffer_atr': [0.0, 0.10], # 2 values - START FROM 0 (no filter)
|
||||
'rsi_long_min': [25, 30], # 2 values - permissive
|
||||
'rsi_short_max': [75, 80], # 2 values - permissive
|
||||
}
|
||||
# Total: 2×4×2×2×2×2×2×2 = 512 combos
|
||||
# Expected: adx_min=0 configs will generate 150-300 signals (proves v11 logic works)
|
||||
# If all still 0 signals with adx_min=0 → base indicator broken, not the filters
|
||||
|
||||
|
||||
def load_market_data(csv_file: str) -> pd.DataFrame:
|
||||
@@ -309,7 +322,7 @@ if __name__ == '__main__':
|
||||
chunk_id = sys.argv[2]
|
||||
start_idx = int(sys.argv[3])
|
||||
|
||||
# Calculate end index (128 combos per chunk)
|
||||
end_idx = start_idx + 128
|
||||
# Calculate end index (256 combos per chunk)
|
||||
end_idx = start_idx + 256
|
||||
|
||||
process_chunk(data_file, chunk_id, start_idx, end_idx)
|
||||
|
||||
Reference in New Issue
Block a user