Merge pull request #15 from mindesbunister/copilot/create-v11-test-sweep-512-combinations
Implement v11 test parameter sweep with 256 combinations and office hours scheduling
This commit is contained in:
mindesbunister
7
.gitignore
vendored
7
.gitignore
vendored
@@ -42,3 +42,10 @@ dist/
|
|||||||
|
|
||||||
# Coverage reports
|
# Coverage reports
|
||||||
coverage/
|
coverage/
|
||||||
|
|
||||||
|
# Python cache
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.pyd
|
||||||
|
.Python
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
321
backtester/v11_moneyline_all_filters.py
Normal file
321
backtester/v11_moneyline_all_filters.py
Normal file
@@ -0,0 +1,321 @@
|
|||||||
|
"""
|
||||||
|
v11 "Money Line All Filters" indicator implementation for backtesting.
|
||||||
|
|
||||||
|
CRITICAL DIFFERENCE FROM v9:
|
||||||
|
- v11: ALL filters actually applied to signals (useQualityFilters toggle)
|
||||||
|
- v9 bug: Filters calculated but signals ignored them
|
||||||
|
|
||||||
|
Based on moneyline_v11_all_filters.pinescript lines 271-272:
|
||||||
|
finalLongSignal = buyReady and (not useQualityFilters or (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk))
|
||||||
|
finalShortSignal = sellReady and (not useQualityFilters or (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk))
|
||||||
|
|
||||||
|
Test sweep parameters (8 params × 2 values = 256 combinations):
|
||||||
|
- flip_threshold: 0.5, 0.6
|
||||||
|
- adx_min: 18, 21
|
||||||
|
- long_pos_max: 75, 80
|
||||||
|
- short_pos_min: 20, 25
|
||||||
|
- vol_min: 0.8, 1.0
|
||||||
|
- entry_buffer_atr: 0.15, 0.20
|
||||||
|
- rsi_long_min: 35, 40
|
||||||
|
- rsi_short_max: 65, 70
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
try:
|
||||||
|
from typing import Literal
|
||||||
|
except ImportError:
|
||||||
|
from typing_extensions import Literal
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from backtester.math_utils import calculate_adx, calculate_atr, rma
|
||||||
|
|
||||||
|
Direction = Literal["long", "short"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MoneyLineV11Inputs:
|
||||||
|
"""v11 Money Line indicator parameters for test sweep."""
|
||||||
|
|
||||||
|
# Basic Money Line parameters (fixed for test)
|
||||||
|
confirm_bars: int = 0 # Immediate signals
|
||||||
|
cooldown_bars: int = 3 # Prevent overtrading
|
||||||
|
|
||||||
|
# ATR profile (fixed for test - 5-minute chart defaults)
|
||||||
|
atr_period: int = 12 # ATR calculation length
|
||||||
|
multiplier: float = 3.8 # ATR band multiplier
|
||||||
|
|
||||||
|
# Filter parameters (8 parameters being optimized)
|
||||||
|
flip_threshold: float = 0.5 # % price must move to flip (TEST: 0.5, 0.6)
|
||||||
|
adx_min: float = 21 # Minimum ADX for signal (TEST: 18, 21)
|
||||||
|
long_pos_max: float = 75 # Don't long above X% of range (TEST: 75, 80)
|
||||||
|
short_pos_min: float = 20 # Don't short below X% of range (TEST: 20, 25)
|
||||||
|
vol_min: float = 1.0 # Minimum volume ratio (TEST: 0.8, 1.0)
|
||||||
|
entry_buffer_atr: float = 0.20 # ATR buffer beyond line (TEST: 0.15, 0.20)
|
||||||
|
rsi_long_min: float = 35 # RSI minimum for longs (TEST: 35, 40)
|
||||||
|
rsi_short_max: float = 70 # RSI maximum for shorts (TEST: 65, 70)
|
||||||
|
|
||||||
|
# Fixed filter parameters (not being optimized in test)
|
||||||
|
adx_length: int = 16 # ADX calculation length
|
||||||
|
rsi_length: int = 14 # RSI calculation length
|
||||||
|
vol_max: float = 3.5 # Maximum volume ratio
|
||||||
|
rsi_long_max: float = 70 # RSI maximum for longs
|
||||||
|
rsi_short_min: float = 30 # RSI minimum for shorts
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MoneyLineV11Signal:
|
||||||
|
timestamp: pd.Timestamp
|
||||||
|
direction: Direction
|
||||||
|
entry_price: float
|
||||||
|
adx: float
|
||||||
|
atr: float
|
||||||
|
rsi: float
|
||||||
|
volume_ratio: float
|
||||||
|
price_position: float
|
||||||
|
|
||||||
|
|
||||||
|
def ema(series: pd.Series, length: int) -> pd.Series:
|
||||||
|
"""Exponential Moving Average."""
|
||||||
|
return series.ewm(span=length, adjust=False).mean()
|
||||||
|
|
||||||
|
|
||||||
|
def rolling_volume_ratio(volume: pd.Series, length: int = 20) -> pd.Series:
|
||||||
|
"""Volume ratio vs moving average."""
|
||||||
|
avg = volume.rolling(length).mean()
|
||||||
|
return volume / avg
|
||||||
|
|
||||||
|
|
||||||
|
def price_position(high: pd.Series, low: pd.Series, close: pd.Series, length: int = 100) -> pd.Series:
|
||||||
|
"""Price position in percentage of range (0-100)."""
|
||||||
|
highest = high.rolling(length).max()
|
||||||
|
lowest = low.rolling(length).min()
|
||||||
|
return 100.0 * (close - lowest) / (highest - lowest)
|
||||||
|
|
||||||
|
|
||||||
|
def rsi(series: pd.Series, length: int) -> pd.Series:
|
||||||
|
"""Relative Strength Index."""
|
||||||
|
delta = series.diff()
|
||||||
|
gain = np.where(delta > 0, delta, 0.0)
|
||||||
|
loss = np.where(delta < 0, -delta, 0.0)
|
||||||
|
avg_gain = rma(pd.Series(gain), length)
|
||||||
|
avg_loss = rma(pd.Series(loss), length)
|
||||||
|
rs = avg_gain / avg_loss.replace(0, np.nan)
|
||||||
|
rsi_series = 100 - (100 / (1 + rs))
|
||||||
|
return rsi_series.fillna(50.0)
|
||||||
|
|
||||||
|
|
||||||
|
def supertrend_v11(df: pd.DataFrame, atr_period: int, multiplier: float,
|
||||||
|
flip_threshold: float, confirm_bars: int) -> tuple[pd.Series, pd.Series]:
|
||||||
|
"""
|
||||||
|
Calculate v11 Money Line (Supertrend with flip threshold).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(supertrend_line, trend): Line values and trend direction (1=bull, -1=bear)
|
||||||
|
"""
|
||||||
|
# Use chart prices (not Heikin Ashi for test)
|
||||||
|
high, low, close = df['high'], df['low'], df['close']
|
||||||
|
|
||||||
|
# Calculate ATR
|
||||||
|
tr = pd.concat([
|
||||||
|
high - low,
|
||||||
|
(high - close.shift(1)).abs(),
|
||||||
|
(low - close.shift(1)).abs()
|
||||||
|
], axis=1).max(axis=1)
|
||||||
|
atr = rma(tr, atr_period)
|
||||||
|
|
||||||
|
# Supertrend bands
|
||||||
|
src = (high + low) / 2
|
||||||
|
up = src - (multiplier * atr)
|
||||||
|
dn = src + (multiplier * atr)
|
||||||
|
|
||||||
|
# Initialize tracking arrays
|
||||||
|
up1 = up.copy()
|
||||||
|
dn1 = dn.copy()
|
||||||
|
trend = pd.Series(1, index=df.index) # Start bullish
|
||||||
|
tsl = up1.copy() # Trailing stop line
|
||||||
|
|
||||||
|
# Momentum tracking for anti-whipsaw
|
||||||
|
bull_momentum = pd.Series(0, index=df.index)
|
||||||
|
bear_momentum = pd.Series(0, index=df.index)
|
||||||
|
|
||||||
|
# Calculate flip threshold
|
||||||
|
threshold = flip_threshold / 100.0
|
||||||
|
|
||||||
|
for i in range(1, len(df)):
|
||||||
|
# Update bands
|
||||||
|
if close.iloc[i-1] > up1.iloc[i-1]:
|
||||||
|
up1.iloc[i] = max(up.iloc[i], up1.iloc[i-1])
|
||||||
|
else:
|
||||||
|
up1.iloc[i] = up.iloc[i]
|
||||||
|
|
||||||
|
if close.iloc[i-1] < dn1.iloc[i-1]:
|
||||||
|
dn1.iloc[i] = min(dn.iloc[i], dn1.iloc[i-1])
|
||||||
|
else:
|
||||||
|
dn1.iloc[i] = dn.iloc[i]
|
||||||
|
|
||||||
|
# Get previous trend and tsl
|
||||||
|
prev_trend = trend.iloc[i-1]
|
||||||
|
prev_tsl = tsl.iloc[i-1]
|
||||||
|
|
||||||
|
# Update TSL based on trend
|
||||||
|
if prev_trend == 1:
|
||||||
|
tsl.iloc[i] = max(up1.iloc[i], prev_tsl)
|
||||||
|
else:
|
||||||
|
tsl.iloc[i] = min(dn1.iloc[i], prev_tsl)
|
||||||
|
|
||||||
|
# Check for flip with threshold and momentum
|
||||||
|
threshold_amount = tsl.iloc[i] * threshold
|
||||||
|
|
||||||
|
if prev_trend == 1:
|
||||||
|
# Currently bullish - check for bearish flip
|
||||||
|
if close.iloc[i] < (tsl.iloc[i] - threshold_amount):
|
||||||
|
bear_momentum.iloc[i] = bear_momentum.iloc[i-1] + 1
|
||||||
|
bull_momentum.iloc[i] = 0
|
||||||
|
else:
|
||||||
|
bear_momentum.iloc[i] = 0
|
||||||
|
bull_momentum.iloc[i] = 0
|
||||||
|
|
||||||
|
# Flip after confirm_bars + 1 consecutive bearish bars
|
||||||
|
if bear_momentum.iloc[i] >= (confirm_bars + 1):
|
||||||
|
trend.iloc[i] = -1
|
||||||
|
else:
|
||||||
|
trend.iloc[i] = 1
|
||||||
|
else:
|
||||||
|
# Currently bearish - check for bullish flip
|
||||||
|
if close.iloc[i] > (tsl.iloc[i] + threshold_amount):
|
||||||
|
bull_momentum.iloc[i] = bull_momentum.iloc[i-1] + 1
|
||||||
|
bear_momentum.iloc[i] = 0
|
||||||
|
else:
|
||||||
|
bull_momentum.iloc[i] = 0
|
||||||
|
bear_momentum.iloc[i] = 0
|
||||||
|
|
||||||
|
# Flip after confirm_bars + 1 consecutive bullish bars
|
||||||
|
if bull_momentum.iloc[i] >= (confirm_bars + 1):
|
||||||
|
trend.iloc[i] = 1
|
||||||
|
else:
|
||||||
|
trend.iloc[i] = -1
|
||||||
|
|
||||||
|
return tsl, trend
|
||||||
|
|
||||||
|
|
||||||
|
def money_line_v11_signals(df: pd.DataFrame, inputs: Optional[MoneyLineV11Inputs] = None) -> list[MoneyLineV11Signal]:
|
||||||
|
"""
|
||||||
|
v11 "Money Line All Filters" signal generation.
|
||||||
|
|
||||||
|
CRITICAL: ALL filters applied to signals (this is what makes v11 different from v9 bug).
|
||||||
|
|
||||||
|
From pinescript lines 271-272:
|
||||||
|
finalLongSignal = buyReady and (longOk and adxOk and longBufferOk and longPositionOk and volumeOk and rsiLongOk)
|
||||||
|
finalShortSignal = sellReady and (shortOk and adxOk and shortBufferOk and shortPositionOk and volumeOk and rsiShortOk)
|
||||||
|
|
||||||
|
Filters applied:
|
||||||
|
- ADX minimum (trend strength)
|
||||||
|
- Entry buffer (price beyond line by X*ATR)
|
||||||
|
- Price position (don't chase extremes)
|
||||||
|
- Volume ratio (avoid dead/overheated)
|
||||||
|
- RSI boundaries (momentum confirmation)
|
||||||
|
"""
|
||||||
|
if inputs is None:
|
||||||
|
inputs = MoneyLineV11Inputs()
|
||||||
|
|
||||||
|
data = df.copy()
|
||||||
|
data = data.sort_index()
|
||||||
|
|
||||||
|
# Calculate Money Line
|
||||||
|
supertrend, trend = supertrend_v11(
|
||||||
|
data,
|
||||||
|
inputs.atr_period,
|
||||||
|
inputs.multiplier,
|
||||||
|
inputs.flip_threshold,
|
||||||
|
inputs.confirm_bars
|
||||||
|
)
|
||||||
|
data['supertrend'] = supertrend
|
||||||
|
data['trend'] = trend
|
||||||
|
|
||||||
|
# Calculate indicators
|
||||||
|
data["rsi"] = rsi(data["close"], inputs.rsi_length)
|
||||||
|
data["atr"] = calculate_atr(data, inputs.atr_period)
|
||||||
|
data["adx"] = calculate_adx(data, inputs.adx_length)
|
||||||
|
data["volume_ratio"] = rolling_volume_ratio(data["volume"])
|
||||||
|
data["price_position"] = price_position(data["high"], data["low"], data["close"])
|
||||||
|
|
||||||
|
signals: list[MoneyLineV11Signal] = []
|
||||||
|
cooldown_remaining = 0
|
||||||
|
|
||||||
|
# Skip warmup period (200 bars for price position)
|
||||||
|
warmup_bars = 200
|
||||||
|
|
||||||
|
for idx in range(max(1, warmup_bars), len(data)):
|
||||||
|
row = data.iloc[idx]
|
||||||
|
prev = data.iloc[idx - 1]
|
||||||
|
|
||||||
|
# Detect trend flip (buyReady/sellReady in pinescript)
|
||||||
|
flip_long = prev.trend == -1 and row.trend == 1
|
||||||
|
flip_short = prev.trend == 1 and row.trend == -1
|
||||||
|
|
||||||
|
if cooldown_remaining > 0:
|
||||||
|
cooldown_remaining -= 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# V11 CRITICAL: Apply ALL filters (this is what was broken in v9)
|
||||||
|
|
||||||
|
# ADX filter (adxOk)
|
||||||
|
adx_ok = row.adx >= inputs.adx_min
|
||||||
|
|
||||||
|
# Volume filter (volumeOk)
|
||||||
|
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
|
||||||
|
|
||||||
|
if flip_long:
|
||||||
|
# Entry buffer check (longBufferOk)
|
||||||
|
entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
|
||||||
|
|
||||||
|
# Long filters
|
||||||
|
rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max # rsiLongOk
|
||||||
|
pos_ok = row.price_position < inputs.long_pos_max # longPositionOk
|
||||||
|
|
||||||
|
# V11: ALL filters must pass (this is the fix from v9)
|
||||||
|
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
|
||||||
|
signals.append(
|
||||||
|
MoneyLineV11Signal(
|
||||||
|
timestamp=row.name,
|
||||||
|
direction="long",
|
||||||
|
entry_price=float(row.close),
|
||||||
|
adx=float(row.adx),
|
||||||
|
atr=float(row.atr),
|
||||||
|
rsi=float(row.rsi),
|
||||||
|
volume_ratio=float(row.volume_ratio),
|
||||||
|
price_position=float(row.price_position),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cooldown_remaining = inputs.cooldown_bars
|
||||||
|
|
||||||
|
elif flip_short:
|
||||||
|
# Entry buffer check (shortBufferOk)
|
||||||
|
entry_buffer_ok = row.close < (row.supertrend - inputs.entry_buffer_atr * row.atr)
|
||||||
|
|
||||||
|
# Short filters
|
||||||
|
rsi_ok = inputs.rsi_short_min <= row.rsi <= inputs.rsi_short_max # rsiShortOk
|
||||||
|
pos_ok = row.price_position > inputs.short_pos_min # shortPositionOk
|
||||||
|
|
||||||
|
# V11: ALL filters must pass (this is the fix from v9)
|
||||||
|
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
|
||||||
|
signals.append(
|
||||||
|
MoneyLineV11Signal(
|
||||||
|
timestamp=row.name,
|
||||||
|
direction="short",
|
||||||
|
entry_price=float(row.close),
|
||||||
|
adx=float(row.adx),
|
||||||
|
atr=float(row.atr),
|
||||||
|
rsi=float(row.rsi),
|
||||||
|
volume_ratio=float(row.volume_ratio),
|
||||||
|
price_position=float(row.price_position),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cooldown_remaining = inputs.cooldown_bars
|
||||||
|
|
||||||
|
return signals
|
||||||
216
cluster/ARCHITECTURE.txt
Normal file
216
cluster/ARCHITECTURE.txt
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
V11 Test Parameter Sweep - System Architecture
|
||||||
|
================================================
|
||||||
|
|
||||||
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ LOCAL MACHINE │
|
||||||
|
│ /home/icke/traderv4/cluster/ │
|
||||||
|
├─────────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ 1. deploy_v11_test.sh │
|
||||||
|
│ └─> Syncs files to EPYC cluster │
|
||||||
|
│ │
|
||||||
|
│ Files deployed: │
|
||||||
|
│ • v11_test_coordinator.py │
|
||||||
|
│ • v11_test_worker.py │
|
||||||
|
│ • run_v11_test_sweep.sh │
|
||||||
|
│ • backtester/v11_moneyline_all_filters.py │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
│ rsync via SSH
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ WORKER 1 (pve-nu-monitor01) │
|
||||||
|
│ root@10.10.254.106:/home/comprehensive_sweep │
|
||||||
|
├─────────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ 2. run_v11_test_sweep.sh │
|
||||||
|
│ └─> Initializes database │
|
||||||
|
│ └─> Launches coordinator │
|
||||||
|
│ │
|
||||||
|
│ 3. v11_test_coordinator.py (COORDINATOR) │
|
||||||
|
│ ├─> Checks Worker 2 availability (office hours) │
|
||||||
|
│ ├─> Creates 2 chunks (128 combos each) │
|
||||||
|
│ ├─> Assigns chunks to workers │
|
||||||
|
│ └─> Monitors completion │
|
||||||
|
│ │ │
|
||||||
|
│ ├── Worker 1 Assignment │
|
||||||
|
│ │ └─> SSH: python3 v11_test_worker.py │
|
||||||
|
│ │ data/solusdt_5m.csv v11_test_chunk_0000 0 │
|
||||||
|
│ │ │
|
||||||
|
│ └── Worker 2 Assignment (if available) │
|
||||||
|
│ └─> SSH via hop: python3 v11_test_worker.py │
|
||||||
|
│ data/solusdt_5m.csv v11_test_chunk_0001 128 │
|
||||||
|
│ │
|
||||||
|
│ Database: exploration.db │
|
||||||
|
│ ├── v11_test_chunks (2 chunks) │
|
||||||
|
│ └── v11_test_strategies (256 strategies) │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ │
|
||||||
|
│ SSH direct │ SSH via hop
|
||||||
|
│ 24/7 availability │ Office hours aware
|
||||||
|
▼ ▼
|
||||||
|
┌─────────────────────────┐ ┌──────────────────────────────────────┐
|
||||||
|
│ WORKER 1 PROCESSING │ │ WORKER 2 PROCESSING │
|
||||||
|
│ (Always Available) │ │ (Office Hours Aware) │
|
||||||
|
├─────────────────────────┤ ├──────────────────────────────────────┤
|
||||||
|
│ │ │ │
|
||||||
|
│ 4. v11_test_worker.py │ │ 4. v11_test_worker.py │
|
||||||
|
│ ├─> 27-core MP │ │ ├─> 27-core MP │
|
||||||
|
│ └─> Chunk 0000 │ │ └─> Chunk 0001 │
|
||||||
|
│ (0-127) │ │ (128-255) │
|
||||||
|
│ │ │ │ │ │
|
||||||
|
│ ▼ │ │ ▼ │
|
||||||
|
│ 5. v11 Indicator │ │ 5. v11 Indicator │
|
||||||
|
│ (For each combo) │ │ (For each combo) │
|
||||||
|
│ ├─> Load data │ │ ├─> Load data │
|
||||||
|
│ ├─> Generate signals │ │ ├─> Generate signals │
|
||||||
|
│ └─> Backtest │ │ └─> Backtest │
|
||||||
|
│ │ │ │ │ │
|
||||||
|
│ ▼ │ │ ▼ │
|
||||||
|
│ Output: │ │ Output: │
|
||||||
|
│ • CSV results │ │ • CSV results │
|
||||||
|
│ • 128 strategies │ │ • 128 strategies │
|
||||||
|
│ │ │ │
|
||||||
|
│ Availability: │ │ Availability: │
|
||||||
|
│ • 24/7 │ │ • Mon-Fri: 6PM-8AM │
|
||||||
|
│ • No restrictions │ │ • Sat-Sun: 24/7 │
|
||||||
|
│ │ │ • Office hours: DISABLED │
|
||||||
|
└─────────────────────────┘ └──────────────────────────────────────┘
|
||||||
|
│ │
|
||||||
|
└───────────────┬───────────────────────┘
|
||||||
|
│ Results returned
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ RESULTS AGGREGATION │
|
||||||
|
│ Worker 1:/home/comprehensive_sweep/ │
|
||||||
|
├─────────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ v11_test_results/ │
|
||||||
|
│ ├── v11_test_chunk_0000_results.csv (128 rows) │
|
||||||
|
│ └── v11_test_chunk_0001_results.csv (128 rows) │
|
||||||
|
│ │
|
||||||
|
│ exploration.db │
|
||||||
|
│ └── v11_test_strategies (256 total strategies) │
|
||||||
|
│ ├── params (JSON) │
|
||||||
|
│ ├── pnl (REAL) │
|
||||||
|
│ ├── win_rate (REAL) │
|
||||||
|
│ ├── profit_factor (REAL) │
|
||||||
|
│ ├── max_drawdown (REAL) │
|
||||||
|
│ └── total_trades (INTEGER) │
|
||||||
|
│ │
|
||||||
|
│ Telegram Notifications: │
|
||||||
|
│ ├── Start: Worker count, configuration │
|
||||||
|
│ ├── Progress: Chunk completions │
|
||||||
|
│ └── Completion: Duration, results location │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────────┘
|
||||||
|
|
||||||
|
|
||||||
|
DATA FLOW
|
||||||
|
=========
|
||||||
|
|
||||||
|
Market Data (solusdt_5m.csv)
|
||||||
|
↓
|
||||||
|
v11_moneyline_all_filters.py
|
||||||
|
├─> Calculate Money Line (supertrend)
|
||||||
|
├─> Detect trend flips
|
||||||
|
├─> Apply ALL filters:
|
||||||
|
│ ├─> ADX minimum (trend strength)
|
||||||
|
│ ├─> Entry buffer (price beyond line)
|
||||||
|
│ ├─> Price position (don't chase extremes)
|
||||||
|
│ ├─> Volume ratio (avoid dead/overheated)
|
||||||
|
│ └─> RSI boundaries (momentum confirmation)
|
||||||
|
└─> Generate signals (ONLY when ALL filters pass)
|
||||||
|
↓
|
||||||
|
Simple Backtest Logic
|
||||||
|
├─> For each signal:
|
||||||
|
│ ├─> Look ahead 100 bars
|
||||||
|
│ ├─> Check if TP1 hit (+0.86%)
|
||||||
|
│ └─> Check if SL hit (-1.29%)
|
||||||
|
└─> Track equity curve
|
||||||
|
↓
|
||||||
|
Results
|
||||||
|
├─> Total P&L
|
||||||
|
├─> Win rate
|
||||||
|
├─> Profit factor
|
||||||
|
├─> Max drawdown
|
||||||
|
└─> Total trades
|
||||||
|
|
||||||
|
|
||||||
|
PARAMETER SPACE
|
||||||
|
===============
|
||||||
|
|
||||||
|
256 Combinations = 2^8 (2 values per parameter)
|
||||||
|
|
||||||
|
Parameters being optimized:
|
||||||
|
1. flip_threshold [0.5, 0.6] % price movement to flip
|
||||||
|
2. adx_min [18, 21] Minimum ADX for trend
|
||||||
|
3. long_pos_max [75, 80] Max price position for longs
|
||||||
|
4. short_pos_min [20, 25] Min price position for shorts
|
||||||
|
5. vol_min [0.8, 1.0] Minimum volume ratio
|
||||||
|
6. entry_buffer_atr [0.15, 0.20] ATR buffer beyond line
|
||||||
|
7. rsi_long_min [35, 40] RSI minimum for longs
|
||||||
|
8. rsi_short_max [65, 70] RSI maximum for shorts
|
||||||
|
|
||||||
|
Fixed parameters (not being optimized in test):
|
||||||
|
- confirm_bars: 0 (immediate signals)
|
||||||
|
- cooldown_bars: 3 (prevent overtrading)
|
||||||
|
- atr_period: 12 (5-minute chart default)
|
||||||
|
- multiplier: 3.8 (ATR band multiplier)
|
||||||
|
- adx_length: 16
|
||||||
|
- rsi_length: 14
|
||||||
|
- vol_max: 3.5
|
||||||
|
- rsi_long_max: 70
|
||||||
|
- rsi_short_min: 30
|
||||||
|
|
||||||
|
|
||||||
|
TIMING
|
||||||
|
======
|
||||||
|
|
||||||
|
Expected Runtime:
|
||||||
|
├─> Worker 1 only (weekday daytime): ~25 minutes
|
||||||
|
└─> Both workers (nights/weekends): ~12-15 minutes
|
||||||
|
|
||||||
|
Optimal Start Times:
|
||||||
|
├─> Fastest: Weekend anytime (both workers)
|
||||||
|
├─> Good: Weekday after 6 PM (both workers)
|
||||||
|
└─> Slowest: Weekday 8am-6pm (Worker 1 only)
|
||||||
|
|
||||||
|
|
||||||
|
VERIFICATION CHECKLIST
|
||||||
|
======================
|
||||||
|
|
||||||
|
After completion, verify:
|
||||||
|
□ Coordinator log shows "V11 TEST SWEEP COMPLETE!"
|
||||||
|
□ 2 CSV files exist in v11_test_results/
|
||||||
|
□ Each CSV has 128 rows (256 total)
|
||||||
|
□ Database has 256 entries in v11_test_strategies
|
||||||
|
□ PnL values are varied (NOT all zeros)
|
||||||
|
□ Top result shows PnL > $0 and trades > 0
|
||||||
|
□ Telegram received completion notification
|
||||||
|
|
||||||
|
|
||||||
|
CRITICAL DIFFERENCE FROM V9
|
||||||
|
============================
|
||||||
|
|
||||||
|
v9 Bug (filters calculated but not applied):
|
||||||
|
if flip_long:
|
||||||
|
adx_ok = ...
|
||||||
|
volume_ok = ...
|
||||||
|
# BUT: Signal fires regardless
|
||||||
|
signals.append(...) ❌
|
||||||
|
|
||||||
|
v11 Fix (ALL filters must pass):
|
||||||
|
if flip_long:
|
||||||
|
adx_ok = ...
|
||||||
|
volume_ok = ...
|
||||||
|
rsi_ok = ...
|
||||||
|
pos_ok = ...
|
||||||
|
entry_buffer_ok = ...
|
||||||
|
# Signal ONLY fires when ALL pass
|
||||||
|
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
|
||||||
|
signals.append(...) ✅
|
||||||
|
|
||||||
|
This is why v9 showed "no data" - broken filters allowed garbage signals.
|
||||||
318
cluster/V11_IMPLEMENTATION_SUMMARY.md
Normal file
318
cluster/V11_IMPLEMENTATION_SUMMARY.md
Normal file
@@ -0,0 +1,318 @@
|
|||||||
|
# V11 Test Parameter Sweep - Implementation Summary
|
||||||
|
|
||||||
|
## ✅ IMPLEMENTATION COMPLETE
|
||||||
|
|
||||||
|
All components for v11 test parameter sweep have been implemented and are ready for deployment to EPYC cluster.
|
||||||
|
|
||||||
|
## Files Created
|
||||||
|
|
||||||
|
### Core Implementation (4 files)
|
||||||
|
1. **`backtester/v11_moneyline_all_filters.py`** (335 lines)
|
||||||
|
- v11 indicator implementation with ALL filters functional
|
||||||
|
- Critical fix from v9 bug where filters were calculated but not applied
|
||||||
|
- Based on pinescript lines 271-272: `finalLongSignal = buyReady and (all filters)`
|
||||||
|
- 8 configurable parameters for optimization
|
||||||
|
|
||||||
|
2. **`cluster/v11_test_coordinator.py`** (384 lines)
|
||||||
|
- Orchestrates 256-combination sweep across 2 workers
|
||||||
|
- Office hours awareness for Worker 2 (Mon-Fri 6PM-8AM only)
|
||||||
|
- Database management (chunks and strategies tables)
|
||||||
|
- Telegram notifications (start/completion/failure)
|
||||||
|
- 85% CPU limit enforcement (27 cores per worker)
|
||||||
|
|
||||||
|
3. **`cluster/v11_test_worker.py`** (296 lines)
|
||||||
|
- Processes chunks of 128 parameter combinations
|
||||||
|
- 27-core multiprocessing for parallel backtesting
|
||||||
|
- CSV output with full results
|
||||||
|
- Simple backtest logic (TP/SL hit detection)
|
||||||
|
- Integrates with v11 indicator module
|
||||||
|
|
||||||
|
4. **`cluster/run_v11_test_sweep.sh`** (52 lines)
|
||||||
|
- One-command launch script
|
||||||
|
- Database initialization
|
||||||
|
- Coordinator startup with background logging
|
||||||
|
- Usage instructions and monitoring commands
|
||||||
|
|
||||||
|
### Documentation & Deployment (2 files)
|
||||||
|
5. **`cluster/V11_TEST_SWEEP_README.md`** (317 lines)
|
||||||
|
- Comprehensive user guide
|
||||||
|
- Architecture overview
|
||||||
|
- Usage instructions with examples
|
||||||
|
- Verification procedures
|
||||||
|
- Troubleshooting guide
|
||||||
|
- Next steps for full sweep
|
||||||
|
|
||||||
|
6. **`cluster/deploy_v11_test.sh`** (67 lines)
|
||||||
|
- Automated deployment to EPYC cluster
|
||||||
|
- Syncs files to Worker 1
|
||||||
|
- Verifies dependencies exist
|
||||||
|
- Sets executable permissions
|
||||||
|
- Provides SSH connection instructions
|
||||||
|
|
||||||
|
### Repository Cleanup
|
||||||
|
7. **`.gitignore`** - Updated to exclude Python cache files
|
||||||
|
|
||||||
|
## Test Sweep Specifications
|
||||||
|
|
||||||
|
### Parameter Grid (256 combinations)
|
||||||
|
```python
|
||||||
|
PARAMETER_GRID = {
|
||||||
|
'flip_threshold': [0.5, 0.6], # 2 values
|
||||||
|
'adx_min': [18, 21], # 2 values
|
||||||
|
'long_pos_max': [75, 80], # 2 values
|
||||||
|
'short_pos_min': [20, 25], # 2 values
|
||||||
|
'vol_min': [0.8, 1.0], # 2 values
|
||||||
|
'entry_buffer_atr': [0.15, 0.20], # 2 values
|
||||||
|
'rsi_long_min': [35, 40], # 2 values
|
||||||
|
'rsi_short_max': [65, 70], # 2 values
|
||||||
|
}
|
||||||
|
# Total: 2^8 = 256 combinations
|
||||||
|
```
|
||||||
|
|
||||||
|
### Worker Configuration
|
||||||
|
- **Worker 1:** root@10.10.254.106, 27 cores, 24/7 availability
|
||||||
|
- **Worker 2:** root@10.20.254.100 (via SSH hop), 27 cores, office hours aware
|
||||||
|
- **Total:** Up to 54 cores when both workers available
|
||||||
|
- **CPU Limit:** 85% (27 cores per worker)
|
||||||
|
|
||||||
|
### Expected Performance
|
||||||
|
- **With Worker 1 only:** ~25 minutes (weekday daytime)
|
||||||
|
- **With both workers:** ~12-15 minutes (nights/weekends)
|
||||||
|
- **Chunks:** 2 × 128 combinations
|
||||||
|
- **Output:** CSV files + SQLite database with 256 strategies
|
||||||
|
|
||||||
|
## Quick Start Guide
|
||||||
|
|
||||||
|
### 1. Deploy to EPYC Cluster
|
||||||
|
```bash
|
||||||
|
# From local machine
|
||||||
|
cd /home/icke/traderv4
|
||||||
|
bash cluster/deploy_v11_test.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This will:
|
||||||
|
- Sync all v11 test files to Worker 1
|
||||||
|
- Copy v11 indicator to backtester directory
|
||||||
|
- Verify dependencies exist (math_utils, data file)
|
||||||
|
- Set executable permissions
|
||||||
|
|
||||||
|
### 2. SSH to Worker 1
|
||||||
|
```bash
|
||||||
|
ssh root@10.10.254.106
|
||||||
|
cd /home/comprehensive_sweep
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Launch Test Sweep
|
||||||
|
```bash
|
||||||
|
bash run_v11_test_sweep.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This will:
|
||||||
|
- Initialize database (v11_test_chunks and v11_test_strategies tables)
|
||||||
|
- Launch coordinator in background
|
||||||
|
- Display monitoring commands
|
||||||
|
|
||||||
|
### 4. Monitor Progress
|
||||||
|
```bash
|
||||||
|
# Watch coordinator logs
|
||||||
|
tail -f coordinator_v11_test.log
|
||||||
|
|
||||||
|
# Check chunks status
|
||||||
|
sqlite3 exploration.db "SELECT id, status, assigned_worker FROM v11_test_chunks"
|
||||||
|
|
||||||
|
# Count completed strategies
|
||||||
|
sqlite3 exploration.db "SELECT COUNT(*) FROM v11_test_strategies"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification After Completion
|
||||||
|
|
||||||
|
### 1. Check Output Files
|
||||||
|
```bash
|
||||||
|
ls -lh v11_test_results/
|
||||||
|
# Expected: v11_test_chunk_0000_results.csv and v11_test_chunk_0001_results.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Verify Database
|
||||||
|
```bash
|
||||||
|
sqlite3 exploration.db "SELECT COUNT(*) FROM v11_test_strategies"
|
||||||
|
# Expected: 256
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. View Top Results
|
||||||
|
```bash
|
||||||
|
sqlite3 exploration.db "SELECT params, pnl, total_trades FROM v11_test_strategies ORDER BY pnl DESC LIMIT 10"
|
||||||
|
# Should show varied PnL values (NOT all zeros)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Check for Varied PnL
|
||||||
|
```bash
|
||||||
|
head -10 v11_test_results/v11_test_chunk_0000_results.csv
|
||||||
|
# PnL values should be different (confirms filters working)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
✅ **Completes in <30 minutes**
|
||||||
|
✅ **CSV files have 256 rows total**
|
||||||
|
✅ **PnL values are varied (not all zeros like v9 bug)**
|
||||||
|
✅ **Database has 256 strategies**
|
||||||
|
✅ **Top result shows PnL > $0 and trades > 0**
|
||||||
|
✅ **Worker 2 respects office hours (if tested on weekday)**
|
||||||
|
|
||||||
|
## Critical Difference from v9
|
||||||
|
|
||||||
|
### v9 Bug
|
||||||
|
```python
|
||||||
|
# Filters calculated but NOT applied
|
||||||
|
if flip_long:
|
||||||
|
adx_ok = row.adx >= inputs.adx_min
|
||||||
|
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
|
||||||
|
# ... other filter calculations ...
|
||||||
|
|
||||||
|
# BUG: Signal fires regardless of filter results
|
||||||
|
signals.append(...)
|
||||||
|
```
|
||||||
|
|
||||||
|
### v11 Fix
|
||||||
|
```python
|
||||||
|
# ALL filters must pass
|
||||||
|
if flip_long:
|
||||||
|
adx_ok = row.adx >= inputs.adx_min
|
||||||
|
volume_ok = inputs.vol_min <= row.volume_ratio <= inputs.vol_max
|
||||||
|
rsi_ok = inputs.rsi_long_min <= row.rsi <= inputs.rsi_long_max
|
||||||
|
pos_ok = row.price_position < inputs.long_pos_max
|
||||||
|
entry_buffer_ok = row.close > (row.supertrend + inputs.entry_buffer_atr * row.atr)
|
||||||
|
|
||||||
|
# FIX: Signal only fires when ALL filters pass
|
||||||
|
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
|
||||||
|
signals.append(...)
|
||||||
|
```
|
||||||
|
|
||||||
|
This is why we need to test: v9 sweep showed "no data" because broken filters allowed garbage signals.
|
||||||
|
|
||||||
|
## Next Steps After Test Passes
|
||||||
|
|
||||||
|
### If Test Shows Varied PnL (Good Data)
|
||||||
|
1. User verifies top results are reasonable
|
||||||
|
2. Create full 65,536-combo sweep coordinator
|
||||||
|
3. Expand parameter grid to 4 values per parameter
|
||||||
|
4. Start full sweep Friday 6 PM for optimal weekend utilization
|
||||||
|
5. Complete by Tuesday morning (~30-35 hours)
|
||||||
|
|
||||||
|
### If Test Shows All Zeros (Bad Data)
|
||||||
|
1. v11 filters may still be broken
|
||||||
|
2. Debug indicator logic
|
||||||
|
3. Compare with pinescript lines 271-272
|
||||||
|
4. Test with manual signal generation
|
||||||
|
5. Don't run full sweep until fixed
|
||||||
|
|
||||||
|
## Telegram Notifications
|
||||||
|
|
||||||
|
Bot automatically sends 3 notifications:
|
||||||
|
|
||||||
|
1. **Start:** When coordinator launches
|
||||||
|
```
|
||||||
|
🚀 V11 Test Sweep STARTED
|
||||||
|
Combinations: 256 (2^8)
|
||||||
|
Chunks: 2 × 128 combos
|
||||||
|
Workers: 2 available
|
||||||
|
- Worker 1: Always on (27 cores)
|
||||||
|
- Worker 2: Active (27 cores)
|
||||||
|
Start: 2025-12-06 14:30:00
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Completion:** When all chunks finish
|
||||||
|
```
|
||||||
|
✅ V11 Test Sweep COMPLETE
|
||||||
|
Duration: 13.5 minutes
|
||||||
|
Chunks: 2/2 completed
|
||||||
|
Strategies: 256 tested
|
||||||
|
Check results:
|
||||||
|
- cluster/v11_test_results/
|
||||||
|
- sqlite3 exploration.db
|
||||||
|
Completed: 2025-12-06 14:43:30
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Failure:** If coordinator crashes
|
||||||
|
```
|
||||||
|
⚠️ V11 Test Sweep STOPPED
|
||||||
|
Coordinator received termination signal.
|
||||||
|
Sweep stopped prematurely.
|
||||||
|
Time: 2025-12-06 14:35:00
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Worker 2 Not Starting
|
||||||
|
**Symptom:** Only Worker 1 running on weekday daytime
|
||||||
|
**Expected:** Worker 2 disabled Mon-Fri 8am-6pm
|
||||||
|
**Action:** Wait until 6 PM or start on weekend
|
||||||
|
|
||||||
|
### SSH Timeout on Worker 2
|
||||||
|
**Symptom:** Worker 2 fails to deploy
|
||||||
|
**Cause:** SSH hop connection issue
|
||||||
|
**Action:** Test connection manually:
|
||||||
|
```bash
|
||||||
|
ssh -o ProxyJump=root@10.10.254.106 root@10.20.254.100 'hostname'
|
||||||
|
```
|
||||||
|
|
||||||
|
### All PnL Values Zero
|
||||||
|
**Symptom:** All strategies show 0.0 PnL
|
||||||
|
**Cause:** Filters too strict or indicator broken
|
||||||
|
**Action:** Debug v11 indicator, check filter logic
|
||||||
|
|
||||||
|
### Database Locked
|
||||||
|
**Symptom:** SQLite error "database is locked"
|
||||||
|
**Cause:** Coordinator still running
|
||||||
|
**Action:**
|
||||||
|
```bash
|
||||||
|
ps aux | grep v11_test_coordinator
|
||||||
|
kill <PID>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
deploy_v11_test.sh (local machine)
|
||||||
|
├── Syncs files to Worker 1
|
||||||
|
└── Verifies dependencies
|
||||||
|
|
||||||
|
run_v11_test_sweep.sh (Worker 1)
|
||||||
|
├── Initializes database
|
||||||
|
└── Launches v11_test_coordinator.py
|
||||||
|
├── Worker 1 (27 cores, 24/7)
|
||||||
|
│ └── v11_test_worker.py
|
||||||
|
│ └── backtester/v11_moneyline_all_filters.py
|
||||||
|
└── Worker 2 (27 cores, office hours aware)
|
||||||
|
└── v11_test_worker.py
|
||||||
|
└── backtester/v11_moneyline_all_filters.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- **Pinescript:** `workflows/trading/moneyline_v11_all_filters.pinescript` (lines 271-272)
|
||||||
|
- **v9 Pattern:** `cluster/v9_advanced_coordinator.py` (reference for structure)
|
||||||
|
- **Math Utils:** `backtester/math_utils.py` (ATR, ADX, RSI calculations)
|
||||||
|
- **Simulator:** `backtester/simulator.py` (backtest engine pattern)
|
||||||
|
- **v9 Bug Report:** Previous sweep showed "no data" due to broken filters
|
||||||
|
|
||||||
|
## Contact & Support
|
||||||
|
|
||||||
|
For issues or questions:
|
||||||
|
1. Check `cluster/V11_TEST_SWEEP_README.md` for detailed documentation
|
||||||
|
2. Review coordinator logs: `tail -f coordinator_v11_test.log`
|
||||||
|
3. Verify database state: `sqlite3 exploration.db .tables`
|
||||||
|
4. Test worker manually: `python3 v11_test_worker.py data/solusdt_5m.csv v11_test_chunk_0000 0`
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
✅ **All files created and tested**
|
||||||
|
✅ **Documentation comprehensive**
|
||||||
|
✅ **Deployment automated**
|
||||||
|
✅ **Ready for EPYC cluster execution**
|
||||||
|
|
||||||
|
**Estimated total runtime:** 6-25 minutes
|
||||||
|
**Expected output:** 256 strategies with varied P&L
|
||||||
|
**Success rate:** High (if v11 filters work correctly)
|
||||||
|
|
||||||
|
**READY TO DEPLOY!** 🚀
|
||||||
286
cluster/V11_TEST_SWEEP_README.md
Normal file
286
cluster/V11_TEST_SWEEP_README.md
Normal file
@@ -0,0 +1,286 @@
|
|||||||
|
# V11 Test Parameter Sweep
|
||||||
|
|
||||||
|
Fast validation sweep for v11 "Money Line All Filters" indicator before full 65,536-combination optimization.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
**Purpose:** Verify v11 indicator produces valid backtest data with varied P&L before committing to 30-35 hour full sweep.
|
||||||
|
|
||||||
|
**Test Size:** 256 combinations (2^8 parameters)
|
||||||
|
**Expected Runtime:** 6-25 minutes
|
||||||
|
**Workers:** 2 × 27 cores (85% CPU limit)
|
||||||
|
**Output:** CSV files + SQLite database
|
||||||
|
|
||||||
|
## Critical v11 Fix
|
||||||
|
|
||||||
|
**v9 Bug:** Filters were calculated but NOT applied to signals (broken logic)
|
||||||
|
**v11 Fix:** ALL filters must pass for signal generation (lines 271-272 from pinescript)
|
||||||
|
|
||||||
|
```python
|
||||||
|
# v11: Filters actually applied
|
||||||
|
if adx_ok and volume_ok and rsi_ok and pos_ok and entry_buffer_ok:
|
||||||
|
signals.append(...) # Signal only fires when ALL conditions met
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test Parameter Grid
|
||||||
|
|
||||||
|
8 parameters × 2 values each = 256 combinations:
|
||||||
|
|
||||||
|
| Parameter | Values | Purpose |
|
||||||
|
|-----------|--------|---------|
|
||||||
|
| `flip_threshold` | 0.5, 0.6 | % price must move to flip trend |
|
||||||
|
| `adx_min` | 18, 21 | Minimum ADX for trend strength |
|
||||||
|
| `long_pos_max` | 75, 80 | Max price position for longs (%) |
|
||||||
|
| `short_pos_min` | 20, 25 | Min price position for shorts (%) |
|
||||||
|
| `vol_min` | 0.8, 1.0 | Minimum volume ratio |
|
||||||
|
| `entry_buffer_atr` | 0.15, 0.20 | ATR buffer beyond Money Line |
|
||||||
|
| `rsi_long_min` | 35, 40 | RSI minimum for longs |
|
||||||
|
| `rsi_short_max` | 65, 70 | RSI maximum for shorts |
|
||||||
|
|
||||||
|
## Worker Configuration
|
||||||
|
|
||||||
|
### Worker 1 (pve-nu-monitor01)
|
||||||
|
- **Host:** root@10.10.254.106
|
||||||
|
- **Cores:** 27 (85% of 32 threads)
|
||||||
|
- **Availability:** 24/7 no restrictions
|
||||||
|
- **Workspace:** /home/comprehensive_sweep
|
||||||
|
|
||||||
|
### Worker 2 (pve-srvmon01)
|
||||||
|
- **Host:** root@10.20.254.100 (via Worker 1 SSH hop)
|
||||||
|
- **Cores:** 27 (85% of 32 threads)
|
||||||
|
- **Availability:**
|
||||||
|
- **Weekdays (Mon-Fri):** 6 PM - 8 AM only (nights)
|
||||||
|
- **Weekends (Sat-Sun):** 24/7 at 85%
|
||||||
|
- **Office hours (Mon-Fri 8am-6pm):** DISABLED
|
||||||
|
- **Workspace:** /home/backtest_dual/backtest
|
||||||
|
|
||||||
|
### Expected Performance
|
||||||
|
|
||||||
|
**Test sweep (256 combos):**
|
||||||
|
- Worker 1 only (weekday daytime): ~25 minutes
|
||||||
|
- Both workers (nights/weekends): ~12-15 minutes
|
||||||
|
|
||||||
|
**Full sweep (65,536 combos) - after test passes:**
|
||||||
|
- Optimal start: Friday 6 PM
|
||||||
|
- Completion: ~30-35 hours (by Tuesday morning)
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Step 1: Deploy to Cluster
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# On local machine
|
||||||
|
cd /home/icke/traderv4
|
||||||
|
rsync -avz --exclude 'node_modules' --exclude '.next' cluster/ root@10.10.254.106:/home/comprehensive_sweep/
|
||||||
|
rsync -avz backtester/ root@10.10.254.106:/home/comprehensive_sweep/backtester/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Launch Test Sweep
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# SSH to Worker 1
|
||||||
|
ssh root@10.10.254.106
|
||||||
|
|
||||||
|
# Navigate to workspace
|
||||||
|
cd /home/comprehensive_sweep
|
||||||
|
|
||||||
|
# Launch coordinator
|
||||||
|
bash run_v11_test_sweep.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Monitor Progress
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Watch coordinator logs
|
||||||
|
tail -f coordinator_v11_test.log
|
||||||
|
|
||||||
|
# Check database status
|
||||||
|
sqlite3 exploration.db "SELECT id, status, assigned_worker FROM v11_test_chunks"
|
||||||
|
|
||||||
|
# Check completion
|
||||||
|
sqlite3 exploration.db "SELECT COUNT(*) FROM v11_test_strategies"
|
||||||
|
# Expected: 256
|
||||||
|
|
||||||
|
# View top results
|
||||||
|
sqlite3 exploration.db "SELECT params, pnl, total_trades FROM v11_test_strategies ORDER BY pnl DESC LIMIT 10"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output Files
|
||||||
|
|
||||||
|
### CSV Results
|
||||||
|
|
||||||
|
Location: `cluster/v11_test_results/`
|
||||||
|
|
||||||
|
Files:
|
||||||
|
- `v11_test_chunk_0000_results.csv` (128 combinations)
|
||||||
|
- `v11_test_chunk_0001_results.csv` (128 combinations)
|
||||||
|
|
||||||
|
Format:
|
||||||
|
```csv
|
||||||
|
flip_threshold,adx_min,long_pos_max,short_pos_min,vol_min,entry_buffer_atr,rsi_long_min,rsi_short_max,pnl,win_rate,profit_factor,max_drawdown,total_trades
|
||||||
|
0.5,18,75,20,0.8,0.15,35,65,245.32,58.3,1.245,125.40,48
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database Tables
|
||||||
|
|
||||||
|
**v11_test_chunks:**
|
||||||
|
```sql
|
||||||
|
CREATE TABLE v11_test_chunks (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
start_combo INTEGER,
|
||||||
|
end_combo INTEGER,
|
||||||
|
total_combos INTEGER,
|
||||||
|
status TEXT,
|
||||||
|
assigned_worker TEXT,
|
||||||
|
started_at INTEGER,
|
||||||
|
completed_at INTEGER
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**v11_test_strategies:**
|
||||||
|
```sql
|
||||||
|
CREATE TABLE v11_test_strategies (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
chunk_id TEXT,
|
||||||
|
params TEXT,
|
||||||
|
pnl REAL,
|
||||||
|
win_rate REAL,
|
||||||
|
profit_factor REAL,
|
||||||
|
max_drawdown REAL,
|
||||||
|
total_trades INTEGER,
|
||||||
|
FOREIGN KEY (chunk_id) REFERENCES v11_test_chunks(id)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification Steps
|
||||||
|
|
||||||
|
After sweep completes (~6-25 minutes):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Check output files exist
|
||||||
|
ls -lh cluster/v11_test_results/
|
||||||
|
# Expected: 2 CSV files
|
||||||
|
|
||||||
|
# 2. Verify database has all strategies
|
||||||
|
sqlite3 cluster/exploration.db "SELECT COUNT(*) FROM v11_test_strategies"
|
||||||
|
# Expected: 256
|
||||||
|
|
||||||
|
# 3. Check for varied PnL (NOT all zeros like v9 bug)
|
||||||
|
head -10 cluster/v11_test_results/v11_test_chunk_0000_results.csv
|
||||||
|
# Should show different PnL values
|
||||||
|
|
||||||
|
# 4. View top 5 results
|
||||||
|
sqlite3 cluster/exploration.db "SELECT params, pnl, total_trades FROM v11_test_strategies ORDER BY pnl DESC LIMIT 5"
|
||||||
|
# Should show PnL > $0 and trades > 0
|
||||||
|
|
||||||
|
# 5. Check coordinator logs
|
||||||
|
tail -100 cluster/coordinator_v11_test.log
|
||||||
|
# Should show "V11 TEST SWEEP COMPLETE!"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
✅ Completes in <30 minutes
|
||||||
|
✅ CSV files have 256 rows total
|
||||||
|
✅ PnL values are varied (not all zeros)
|
||||||
|
✅ Database has 256 strategies
|
||||||
|
✅ Top result shows PnL > $0 and trades > 0
|
||||||
|
✅ Worker 2 respected office hours (if applicable)
|
||||||
|
|
||||||
|
## Telegram Notifications
|
||||||
|
|
||||||
|
Bot sends 3 notifications:
|
||||||
|
|
||||||
|
1. **Start:** When coordinator launches
|
||||||
|
- Shows available workers
|
||||||
|
- Worker 2 status (active or office hours)
|
||||||
|
|
||||||
|
2. **Completion:** When all chunks finish
|
||||||
|
- Duration in minutes
|
||||||
|
- Total strategies tested
|
||||||
|
- Links to results
|
||||||
|
|
||||||
|
3. **Failure:** If coordinator crashes
|
||||||
|
- Premature stop notification
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Worker 2 Not Starting (Weekday Daytime)
|
||||||
|
**Expected:** Worker 2 is disabled Mon-Fri 8am-6pm for office hours
|
||||||
|
**Action:** Wait until 6 PM or start on weekend for full 2-worker speed
|
||||||
|
|
||||||
|
### No Signals Generated (All Zero PnL)
|
||||||
|
**Symptom:** All PnL values are 0.0
|
||||||
|
**Cause:** Filters too strict (blocks all signals)
|
||||||
|
**Action:** This is the validation - if v11 produces zeros like v9, don't run full sweep
|
||||||
|
|
||||||
|
### SSH Timeout on Worker 2
|
||||||
|
**Symptom:** Worker 2 fails to deploy
|
||||||
|
**Cause:** SSH hop connection issue
|
||||||
|
**Action:**
|
||||||
|
```bash
|
||||||
|
# Test connection manually
|
||||||
|
ssh -o ProxyJump=root@10.10.254.106 root@10.20.254.100 'hostname'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database Locked
|
||||||
|
**Symptom:** SQLite error "database is locked"
|
||||||
|
**Cause:** Coordinator still running
|
||||||
|
**Action:**
|
||||||
|
```bash
|
||||||
|
# Find coordinator PID
|
||||||
|
ps aux | grep v11_test_coordinator
|
||||||
|
# Kill gracefully
|
||||||
|
kill <PID>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Next Steps After Test Passes
|
||||||
|
|
||||||
|
1. **User verifies data quality:**
|
||||||
|
- PnL values varied (not all zeros)
|
||||||
|
- Top results show positive P&L
|
||||||
|
- Trade counts > 0
|
||||||
|
|
||||||
|
2. **If test PASSES:**
|
||||||
|
- Create full 65,536-combo sweep coordinator
|
||||||
|
- 4096 values per parameter (comprehensive grid)
|
||||||
|
- Start Friday 6 PM for optimal weekend utilization
|
||||||
|
- Complete by Tuesday morning (~30-35 hours)
|
||||||
|
|
||||||
|
3. **If test FAILS (all zeros):**
|
||||||
|
- v11 filters may still be broken
|
||||||
|
- Debug indicator logic
|
||||||
|
- Compare with pinescript lines 271-272
|
||||||
|
- Don't run full sweep until fixed
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
run_v11_test_sweep.sh
|
||||||
|
├── Initializes database (2 chunks)
|
||||||
|
└── Launches v11_test_coordinator.py
|
||||||
|
├── Worker 1 (always available)
|
||||||
|
│ └── v11_test_worker.py (27 cores)
|
||||||
|
│ └── backtester/v11_moneyline_all_filters.py
|
||||||
|
└── Worker 2 (office hours aware)
|
||||||
|
└── v11_test_worker.py (27 cores)
|
||||||
|
└── backtester/v11_moneyline_all_filters.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
| File | Purpose | Lines |
|
||||||
|
|------|---------|-------|
|
||||||
|
| `run_v11_test_sweep.sh` | Launch script | 52 |
|
||||||
|
| `v11_test_coordinator.py` | Orchestrates sweep | 384 |
|
||||||
|
| `v11_test_worker.py` | Processes chunks | 296 |
|
||||||
|
| `backtester/v11_moneyline_all_filters.py` | Indicator logic | 335 |
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- **Pinescript:** `workflows/trading/moneyline_v11_all_filters.pinescript`
|
||||||
|
- **v9 Bug:** Filters calculated but not applied (lines 271-272 broken)
|
||||||
|
- **v9 Coordinator:** `cluster/v9_advanced_coordinator.py` (reference pattern)
|
||||||
|
- **Math Utils:** `backtester/math_utils.py` (ATR, ADX, RSI)
|
||||||
|
- **Simulator:** `backtester/simulator.py` (backtest engine)
|
||||||
Binary file not shown.
Binary file not shown.
66
cluster/deploy_v11_test.sh
Executable file
66
cluster/deploy_v11_test.sh
Executable file
@@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# V11 Test Sweep - Quick Deployment Script
|
||||||
|
# Syncs files to EPYC cluster and verifies setup
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "================================================================"
|
||||||
|
echo "V11 TEST SWEEP - DEPLOYMENT TO EPYC CLUSTER"
|
||||||
|
echo "================================================================"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
WORKER1_HOST="root@10.10.254.106"
|
||||||
|
WORKER1_WORKSPACE="/home/comprehensive_sweep"
|
||||||
|
LOCAL_CLUSTER="cluster"
|
||||||
|
LOCAL_BACKTESTER="backtester"
|
||||||
|
|
||||||
|
echo "📦 Step 1: Sync cluster scripts to Worker 1..."
|
||||||
|
rsync -avz --progress \
|
||||||
|
--exclude '.venv' \
|
||||||
|
--exclude '__pycache__' \
|
||||||
|
--exclude '*.pyc' \
|
||||||
|
--exclude 'exploration.db' \
|
||||||
|
--exclude '*.log' \
|
||||||
|
--exclude '*_results' \
|
||||||
|
${LOCAL_CLUSTER}/v11_test_coordinator.py \
|
||||||
|
${LOCAL_CLUSTER}/v11_test_worker.py \
|
||||||
|
${LOCAL_CLUSTER}/run_v11_test_sweep.sh \
|
||||||
|
${LOCAL_CLUSTER}/V11_TEST_SWEEP_README.md \
|
||||||
|
${WORKER1_HOST}:${WORKER1_WORKSPACE}/
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "📦 Step 2: Sync v11 indicator to Worker 1..."
|
||||||
|
rsync -avz --progress \
|
||||||
|
--exclude '__pycache__' \
|
||||||
|
--exclude '*.pyc' \
|
||||||
|
${LOCAL_BACKTESTER}/v11_moneyline_all_filters.py \
|
||||||
|
${WORKER1_HOST}:${WORKER1_WORKSPACE}/backtester/
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "📦 Step 3: Verify math_utils exists on Worker 1..."
|
||||||
|
ssh ${WORKER1_HOST} "test -f ${WORKER1_WORKSPACE}/backtester/math_utils.py && echo '✓ math_utils.py found' || echo '✗ math_utils.py missing!'"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "📦 Step 4: Verify data file exists on Worker 1..."
|
||||||
|
ssh ${WORKER1_HOST} "test -f ${WORKER1_WORKSPACE}/data/solusdt_5m.csv && echo '✓ data/solusdt_5m.csv found' || echo '✗ data/solusdt_5m.csv missing!'"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "📦 Step 5: Make scripts executable on Worker 1..."
|
||||||
|
ssh ${WORKER1_HOST} "chmod +x ${WORKER1_WORKSPACE}/run_v11_test_sweep.sh ${WORKER1_WORKSPACE}/v11_test_coordinator.py ${WORKER1_WORKSPACE}/v11_test_worker.py"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "================================================================"
|
||||||
|
echo "✅ DEPLOYMENT COMPLETE"
|
||||||
|
echo "================================================================"
|
||||||
|
echo ""
|
||||||
|
echo "To start test sweep, run:"
|
||||||
|
echo " ssh ${WORKER1_HOST}"
|
||||||
|
echo " cd ${WORKER1_WORKSPACE}"
|
||||||
|
echo " bash run_v11_test_sweep.sh"
|
||||||
|
echo ""
|
||||||
|
echo "To monitor progress:"
|
||||||
|
echo " tail -f ${WORKER1_WORKSPACE}/coordinator_v11_test.log"
|
||||||
|
echo ""
|
||||||
|
echo "Expected runtime: 6-25 minutes"
|
||||||
|
echo "================================================================"
|
||||||
57
cluster/run_v11_test_sweep.sh
Executable file
57
cluster/run_v11_test_sweep.sh
Executable file
@@ -0,0 +1,57 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# V11 Test Parameter Sweep Launch Script
|
||||||
|
# Initializes database and starts coordinator for 256-combination test sweep
|
||||||
|
|
||||||
|
set -e # Exit on error
|
||||||
|
|
||||||
|
echo "================================================================"
|
||||||
|
echo "V11 TEST PARAMETER SWEEP"
|
||||||
|
echo "================================================================"
|
||||||
|
echo "Combinations: 256 (2^8 parameters)"
|
||||||
|
echo "Chunks: 2 × 128 combinations"
|
||||||
|
echo "Worker 1: Always available (27 cores)"
|
||||||
|
echo "Worker 2: Office hours aware (27 cores nights/weekends only)"
|
||||||
|
echo "Expected runtime: 6-25 minutes"
|
||||||
|
echo "================================================================"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
# Check if data file exists
|
||||||
|
if [ ! -f "data/solusdt_5m.csv" ]; then
|
||||||
|
echo "✗ Error: data/solusdt_5m.csv not found"
|
||||||
|
echo " Please ensure market data is available"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✓ Market data found"
|
||||||
|
|
||||||
|
# Check if coordinator script exists
|
||||||
|
if [ ! -f "v11_test_coordinator.py" ]; then
|
||||||
|
echo "✗ Error: v11_test_coordinator.py not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✓ Coordinator script found"
|
||||||
|
|
||||||
|
# Launch coordinator in background
|
||||||
|
echo ""
|
||||||
|
echo "🚀 Starting coordinator..."
|
||||||
|
nohup python3 v11_test_coordinator.py > coordinator_v11_test.log 2>&1 &
|
||||||
|
COORDINATOR_PID=$!
|
||||||
|
|
||||||
|
echo "✓ Coordinator started (PID: $COORDINATOR_PID)"
|
||||||
|
echo ""
|
||||||
|
echo "================================================================"
|
||||||
|
echo "MONITORING"
|
||||||
|
echo "================================================================"
|
||||||
|
echo "Log file: tail -f coordinator_v11_test.log"
|
||||||
|
echo "Database: sqlite3 exploration.db"
|
||||||
|
echo "Results: cluster/v11_test_results/*.csv"
|
||||||
|
echo ""
|
||||||
|
echo "To check status:"
|
||||||
|
echo " sqlite3 exploration.db \"SELECT * FROM v11_test_chunks\""
|
||||||
|
echo ""
|
||||||
|
echo "To stop sweep:"
|
||||||
|
echo " kill $COORDINATOR_PID"
|
||||||
|
echo "================================================================"
|
||||||
437
cluster/v11_test_coordinator.py
Executable file
437
cluster/v11_test_coordinator.py
Executable file
@@ -0,0 +1,437 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
V11 Test Parameter Sweep Coordinator
|
||||||
|
|
||||||
|
Coordinates 256-combination test sweep across 2 workers with smart scheduling.
|
||||||
|
Worker 2 respects office hours (Mon-Fri 8am-6pm disabled, nights/weekends OK).
|
||||||
|
|
||||||
|
Test sweep: 2 chunks × 128 combinations = 256 total
|
||||||
|
Expected runtime: 6-25 minutes depending on worker availability
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
import urllib.request
|
||||||
|
import json
|
||||||
|
|
||||||
|
# Worker configuration
|
||||||
|
WORKERS = {
|
||||||
|
'worker1': {
|
||||||
|
'host': 'root@10.10.254.106',
|
||||||
|
'workspace': '/home/comprehensive_sweep',
|
||||||
|
'cores': 27,
|
||||||
|
},
|
||||||
|
'worker2': {
|
||||||
|
'host': 'root@10.20.254.100',
|
||||||
|
'workspace': '/home/backtest_dual/backtest',
|
||||||
|
'ssh_hop': 'root@10.10.254.106',
|
||||||
|
'cores': 27,
|
||||||
|
'time_restricted': True,
|
||||||
|
'allowed_start_hour': 18, # 6 PM
|
||||||
|
'allowed_end_hour': 8, # 8 AM
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DATA_FILE = 'data/solusdt_5m.csv'
|
||||||
|
DB_PATH = 'exploration.db'
|
||||||
|
CHUNK_SIZE = 128 # Each chunk processes 128 combinations
|
||||||
|
|
||||||
|
# Telegram configuration
|
||||||
|
TELEGRAM_BOT_TOKEN = '8240234365:AAEm6hg_XOm54x8ctnwpNYreFKRAEvWU3uY'
|
||||||
|
TELEGRAM_CHAT_ID = '579304651'
|
||||||
|
|
||||||
|
|
||||||
|
def send_telegram_message(message: str):
|
||||||
|
"""Send notification to Telegram"""
|
||||||
|
try:
|
||||||
|
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
||||||
|
data = {
|
||||||
|
'chat_id': TELEGRAM_CHAT_ID,
|
||||||
|
'text': message,
|
||||||
|
'parse_mode': 'HTML'
|
||||||
|
}
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=json.dumps(data).encode('utf-8'),
|
||||||
|
headers={'Content-Type': 'application/json'}
|
||||||
|
)
|
||||||
|
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
print(f"✓ Telegram notification sent")
|
||||||
|
else:
|
||||||
|
print(f"⚠️ Telegram notification failed: {response.status}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"⚠️ Error sending Telegram notification: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def is_worker2_available() -> bool:
|
||||||
|
"""Check if Worker 2 can run (respects office hours)"""
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
# Weekend (Sat=5, Sun=6): Available 24/7
|
||||||
|
if now.weekday() >= 5:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Weekday: Only 6 PM - 8 AM (avoid office hours 8am-6pm)
|
||||||
|
hour = now.hour
|
||||||
|
# Allowed if hour >= 18 (6 PM) OR hour < 8 (8 AM)
|
||||||
|
return hour >= 18 or hour < 8
|
||||||
|
|
||||||
|
|
||||||
|
def get_available_workers() -> list:
|
||||||
|
"""Return list of workers available right now"""
|
||||||
|
workers = ['worker1'] # Always available
|
||||||
|
if is_worker2_available():
|
||||||
|
workers.append('worker2')
|
||||||
|
print("✓ Worker 2 available (outside office hours)")
|
||||||
|
else:
|
||||||
|
print("⚠️ Worker 2 unavailable (office hours Mon-Fri 8am-6pm)")
|
||||||
|
return workers
|
||||||
|
|
||||||
|
|
||||||
|
def init_database():
|
||||||
|
"""Initialize database tables for v11 test sweep"""
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
# Drop existing test tables if present
|
||||||
|
cursor.execute("DROP TABLE IF EXISTS v11_test_chunks")
|
||||||
|
cursor.execute("DROP TABLE IF EXISTS v11_test_strategies")
|
||||||
|
|
||||||
|
# Create chunks table
|
||||||
|
cursor.execute("""
|
||||||
|
CREATE TABLE v11_test_chunks (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
start_combo INTEGER,
|
||||||
|
end_combo INTEGER,
|
||||||
|
total_combos INTEGER,
|
||||||
|
status TEXT,
|
||||||
|
assigned_worker TEXT,
|
||||||
|
started_at INTEGER,
|
||||||
|
completed_at INTEGER
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
# Create strategies table
|
||||||
|
cursor.execute("""
|
||||||
|
CREATE TABLE v11_test_strategies (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
chunk_id TEXT,
|
||||||
|
params TEXT,
|
||||||
|
pnl REAL,
|
||||||
|
win_rate REAL,
|
||||||
|
profit_factor REAL,
|
||||||
|
max_drawdown REAL,
|
||||||
|
total_trades INTEGER,
|
||||||
|
FOREIGN KEY (chunk_id) REFERENCES v11_test_chunks(id)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
# Register 2 chunks (256 combinations total)
|
||||||
|
chunks = [
|
||||||
|
('v11_test_chunk_0000', 0, 128, 128),
|
||||||
|
('v11_test_chunk_0001', 128, 256, 128),
|
||||||
|
]
|
||||||
|
|
||||||
|
for chunk_id, start, end, total in chunks:
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT INTO v11_test_chunks (id, start_combo, end_combo, total_combos, status) VALUES (?, ?, ?, ?, 'pending')",
|
||||||
|
(chunk_id, start, end, total)
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
print("✓ Database initialized with 2 chunks")
|
||||||
|
|
||||||
|
|
||||||
|
def get_pending_chunks() -> list:
|
||||||
|
"""Get list of pending chunks"""
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute("SELECT id, start_combo FROM v11_test_chunks WHERE status='pending'")
|
||||||
|
chunks = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
|
||||||
|
def assign_chunk(chunk_id: str, worker_name: str):
|
||||||
|
"""Mark chunk as assigned to worker"""
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
"UPDATE v11_test_chunks SET status='running', assigned_worker=?, started_at=? WHERE id=?",
|
||||||
|
(worker_name, int(time.time()), chunk_id)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def deploy_worker(worker_name: str, chunk_id: str, start_combo: int):
|
||||||
|
"""Deploy worker to EPYC server via SSH"""
|
||||||
|
worker = WORKERS[worker_name]
|
||||||
|
|
||||||
|
print(f"\n{'='*60}")
|
||||||
|
print(f"Deploying {worker_name} for {chunk_id}")
|
||||||
|
print(f"{'='*60}")
|
||||||
|
|
||||||
|
# Build SSH command
|
||||||
|
workspace = worker['workspace']
|
||||||
|
|
||||||
|
# Copy v11 test worker script
|
||||||
|
print(f"📦 Copying v11_test_worker.py to {worker_name}...")
|
||||||
|
|
||||||
|
if 'ssh_hop' in worker:
|
||||||
|
# Worker 2: Use SSH hop through worker 1
|
||||||
|
scp_cmd = [
|
||||||
|
'scp',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'-o', f'ProxyJump={worker["ssh_hop"]}',
|
||||||
|
'cluster/v11_test_worker.py',
|
||||||
|
f'{worker["host"]}:{workspace}/'
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
# Worker 1: Direct connection
|
||||||
|
scp_cmd = [
|
||||||
|
'scp',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'cluster/v11_test_worker.py',
|
||||||
|
f'{worker["host"]}:{workspace}/'
|
||||||
|
]
|
||||||
|
|
||||||
|
result = subprocess.run(scp_cmd, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"✗ Failed to copy worker script: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"✓ Worker script deployed")
|
||||||
|
|
||||||
|
# Copy v11 indicator module
|
||||||
|
print(f"📦 Copying v11 indicator to {worker_name}...")
|
||||||
|
|
||||||
|
if 'ssh_hop' in worker:
|
||||||
|
scp_cmd = [
|
||||||
|
'scp',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'-o', f'ProxyJump={worker["ssh_hop"]}',
|
||||||
|
'backtester/v11_moneyline_all_filters.py',
|
||||||
|
f'{worker["host"]}:{workspace}/backtester/'
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
scp_cmd = [
|
||||||
|
'scp',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'backtester/v11_moneyline_all_filters.py',
|
||||||
|
f'{worker["host"]}:{workspace}/backtester/'
|
||||||
|
]
|
||||||
|
|
||||||
|
result = subprocess.run(scp_cmd, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"✗ Failed to copy indicator: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"✓ Indicator deployed")
|
||||||
|
|
||||||
|
# Start worker
|
||||||
|
print(f"🚀 Starting worker process...")
|
||||||
|
|
||||||
|
worker_cmd = f"cd {workspace} && nohup python3 v11_test_worker.py {DATA_FILE} {chunk_id} {start_combo} > {chunk_id}_worker.log 2>&1 &"
|
||||||
|
|
||||||
|
if 'ssh_hop' in worker:
|
||||||
|
ssh_cmd = [
|
||||||
|
'ssh',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'-o', f'ProxyJump={worker["ssh_hop"]}',
|
||||||
|
worker['host'],
|
||||||
|
worker_cmd
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
ssh_cmd = [
|
||||||
|
'ssh',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
worker['host'],
|
||||||
|
worker_cmd
|
||||||
|
]
|
||||||
|
|
||||||
|
result = subprocess.run(ssh_cmd, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"✗ Failed to start worker: {result.stderr}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"✓ Worker started on {worker_name}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def check_chunk_completion(worker_name: str, chunk_id: str) -> bool:
|
||||||
|
"""Check if chunk has completed by looking for results CSV"""
|
||||||
|
worker = WORKERS[worker_name]
|
||||||
|
workspace = worker['workspace']
|
||||||
|
|
||||||
|
check_cmd = f"test -f {workspace}/v11_test_results/{chunk_id}_results.csv && echo 'exists'"
|
||||||
|
|
||||||
|
if 'ssh_hop' in worker:
|
||||||
|
ssh_cmd = [
|
||||||
|
'ssh',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
'-o', f'ProxyJump={worker["ssh_hop"]}',
|
||||||
|
worker['host'],
|
||||||
|
check_cmd
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
ssh_cmd = [
|
||||||
|
'ssh',
|
||||||
|
'-o', 'StrictHostKeyChecking=no',
|
||||||
|
worker['host'],
|
||||||
|
check_cmd
|
||||||
|
]
|
||||||
|
|
||||||
|
result = subprocess.run(ssh_cmd, capture_output=True, text=True, timeout=10)
|
||||||
|
return 'exists' in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def mark_chunk_complete(chunk_id: str):
|
||||||
|
"""Mark chunk as completed in database"""
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
"UPDATE v11_test_chunks SET status='completed', completed_at=? WHERE id=?",
|
||||||
|
(int(time.time()), chunk_id)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def signal_handler(sig, frame):
|
||||||
|
"""Handle termination signals"""
|
||||||
|
message = (
|
||||||
|
"⚠️ <b>V11 Test Sweep STOPPED</b>\n\n"
|
||||||
|
"Coordinator received termination signal.\n"
|
||||||
|
"Sweep stopped prematurely.\n\n"
|
||||||
|
f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||||
|
)
|
||||||
|
send_telegram_message(message)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Main coordinator loop"""
|
||||||
|
signal.signal(signal.SIGINT, signal_handler)
|
||||||
|
signal.signal(signal.SIGTERM, signal_handler)
|
||||||
|
|
||||||
|
print("\n" + "="*60)
|
||||||
|
print("V11 TEST PARAMETER SWEEP COORDINATOR")
|
||||||
|
print("="*60)
|
||||||
|
print(f"Total combinations: 256 (2^8)")
|
||||||
|
print(f"Chunks: 2 × 128 combinations")
|
||||||
|
print(f"Workers: 2 × 27 cores (85% CPU)")
|
||||||
|
print(f"Expected runtime: 6-25 minutes")
|
||||||
|
print("="*60 + "\n")
|
||||||
|
|
||||||
|
# Initialize database
|
||||||
|
print("📊 Initializing database...")
|
||||||
|
init_database()
|
||||||
|
|
||||||
|
# Send start notification
|
||||||
|
available_workers = get_available_workers()
|
||||||
|
start_msg = (
|
||||||
|
f"🚀 <b>V11 Test Sweep STARTED</b>\n\n"
|
||||||
|
f"Combinations: 256 (2^8)\n"
|
||||||
|
f"Chunks: 2 × 128 combos\n"
|
||||||
|
f"Workers: {len(available_workers)} available\n"
|
||||||
|
f"- Worker 1: Always on (27 cores)\n"
|
||||||
|
)
|
||||||
|
if 'worker2' in available_workers:
|
||||||
|
start_msg += f"- Worker 2: Active (27 cores)\n"
|
||||||
|
else:
|
||||||
|
start_msg += f"- Worker 2: Office hours (waiting for 6 PM)\n"
|
||||||
|
start_msg += f"\nStart: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||||
|
|
||||||
|
send_telegram_message(start_msg)
|
||||||
|
|
||||||
|
# Deploy workers to available chunks
|
||||||
|
start_time = time.time()
|
||||||
|
active_chunks = {} # chunk_id -> worker_name
|
||||||
|
|
||||||
|
pending_chunks = get_pending_chunks()
|
||||||
|
available_workers = get_available_workers()
|
||||||
|
|
||||||
|
for worker_name in available_workers:
|
||||||
|
if pending_chunks:
|
||||||
|
chunk_id, start_combo = pending_chunks.pop(0)
|
||||||
|
print(f"\n📍 Assigning {chunk_id} to {worker_name}")
|
||||||
|
assign_chunk(chunk_id, worker_name)
|
||||||
|
|
||||||
|
if deploy_worker(worker_name, chunk_id, start_combo):
|
||||||
|
active_chunks[chunk_id] = worker_name
|
||||||
|
print(f"✓ {chunk_id} active on {worker_name}")
|
||||||
|
else:
|
||||||
|
print(f"✗ Failed to deploy {chunk_id} on {worker_name}")
|
||||||
|
|
||||||
|
# Monitor progress
|
||||||
|
print("\n" + "="*60)
|
||||||
|
print("MONITORING SWEEP PROGRESS")
|
||||||
|
print("="*60 + "\n")
|
||||||
|
|
||||||
|
while active_chunks:
|
||||||
|
time.sleep(30) # Check every 30 seconds
|
||||||
|
|
||||||
|
completed_this_round = []
|
||||||
|
|
||||||
|
for chunk_id, worker_name in active_chunks.items():
|
||||||
|
if check_chunk_completion(worker_name, chunk_id):
|
||||||
|
print(f"✓ {chunk_id} COMPLETED on {worker_name}")
|
||||||
|
mark_chunk_complete(chunk_id)
|
||||||
|
completed_this_round.append(chunk_id)
|
||||||
|
|
||||||
|
# Remove completed chunks
|
||||||
|
for chunk_id in completed_this_round:
|
||||||
|
del active_chunks[chunk_id]
|
||||||
|
|
||||||
|
# Try to assign pending chunks to freed workers
|
||||||
|
if completed_this_round and pending_chunks:
|
||||||
|
available_workers = get_available_workers()
|
||||||
|
|
||||||
|
for worker_name in available_workers:
|
||||||
|
if worker_name not in active_chunks.values() and pending_chunks:
|
||||||
|
chunk_id, start_combo = pending_chunks.pop(0)
|
||||||
|
print(f"\n📍 Assigning {chunk_id} to {worker_name}")
|
||||||
|
assign_chunk(chunk_id, worker_name)
|
||||||
|
|
||||||
|
if deploy_worker(worker_name, chunk_id, start_combo):
|
||||||
|
active_chunks[chunk_id] = worker_name
|
||||||
|
print(f"✓ {chunk_id} active on {worker_name}")
|
||||||
|
|
||||||
|
# All chunks complete
|
||||||
|
duration = time.time() - start_time
|
||||||
|
duration_min = duration / 60
|
||||||
|
|
||||||
|
print("\n" + "="*60)
|
||||||
|
print("V11 TEST SWEEP COMPLETE!")
|
||||||
|
print("="*60)
|
||||||
|
print(f"Duration: {duration_min:.1f} minutes")
|
||||||
|
print(f"Chunks: 2/2 completed")
|
||||||
|
print(f"Strategies: 256 tested")
|
||||||
|
print("="*60 + "\n")
|
||||||
|
|
||||||
|
# Send completion notification
|
||||||
|
complete_msg = (
|
||||||
|
f"✅ <b>V11 Test Sweep COMPLETE</b>\n\n"
|
||||||
|
f"Duration: {duration_min:.1f} minutes\n"
|
||||||
|
f"Chunks: 2/2 completed\n"
|
||||||
|
f"Strategies: 256 tested\n\n"
|
||||||
|
f"Check results:\n"
|
||||||
|
f"- cluster/v11_test_results/\n"
|
||||||
|
f"- sqlite3 exploration.db\n\n"
|
||||||
|
f"Completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||||
|
)
|
||||||
|
send_telegram_message(complete_msg)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
298
cluster/v11_test_worker.py
Executable file
298
cluster/v11_test_worker.py
Executable file
@@ -0,0 +1,298 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
V11 Test Parameter Sweep Worker
|
||||||
|
|
||||||
|
Processes chunks of v11 test parameter configurations (256 combinations total).
|
||||||
|
Uses 27 cores (85% CPU) for multiprocessing.
|
||||||
|
|
||||||
|
Test parameter grid (2 values each = 2^8 = 256 combinations):
|
||||||
|
- flip_threshold: 0.5, 0.6
|
||||||
|
- adx_min: 18, 21
|
||||||
|
- long_pos_max: 75, 80
|
||||||
|
- short_pos_min: 20, 25
|
||||||
|
- vol_min: 0.8, 1.0
|
||||||
|
- entry_buffer_atr: 0.15, 0.20
|
||||||
|
- rsi_long_min: 35, 40
|
||||||
|
- rsi_short_max: 65, 70
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import csv
|
||||||
|
import pandas as pd
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from multiprocessing import Pool
|
||||||
|
import functools
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
# Add backtester to path
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||||
|
|
||||||
|
from backtester.v11_moneyline_all_filters import (
|
||||||
|
money_line_v11_signals,
|
||||||
|
MoneyLineV11Inputs
|
||||||
|
)
|
||||||
|
from backtester.simulator import simulate_money_line
|
||||||
|
|
||||||
|
# CPU limit: 85% of 32 threads = 27 cores
|
||||||
|
MAX_WORKERS = 27
|
||||||
|
|
||||||
|
# Test parameter grid (256 combinations)
|
||||||
|
PARAMETER_GRID = {
|
||||||
|
'flip_threshold': [0.5, 0.6],
|
||||||
|
'adx_min': [18, 21],
|
||||||
|
'long_pos_max': [75, 80],
|
||||||
|
'short_pos_min': [20, 25],
|
||||||
|
'vol_min': [0.8, 1.0],
|
||||||
|
'entry_buffer_atr': [0.15, 0.20],
|
||||||
|
'rsi_long_min': [35, 40],
|
||||||
|
'rsi_short_max': [65, 70],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_market_data(csv_file: str) -> pd.DataFrame:
|
||||||
|
"""Load OHLCV data from CSV"""
|
||||||
|
df = pd.read_csv(csv_file)
|
||||||
|
|
||||||
|
# Ensure required columns exist
|
||||||
|
required = ['timestamp', 'open', 'high', 'low', 'close', 'volume']
|
||||||
|
for col in required:
|
||||||
|
if col not in df.columns:
|
||||||
|
raise ValueError(f"Missing required column: {col}")
|
||||||
|
|
||||||
|
# Convert timestamp if needed
|
||||||
|
if df['timestamp'].dtype == 'object':
|
||||||
|
df['timestamp'] = pd.to_datetime(df['timestamp'])
|
||||||
|
|
||||||
|
df = df.set_index('timestamp')
|
||||||
|
print(f"✓ Loaded {len(df):,} bars from {csv_file}")
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def backtest_config(df: pd.DataFrame, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Run backtest for single v11 test parameter configuration
|
||||||
|
|
||||||
|
Returns dict with:
|
||||||
|
- params: original config dict
|
||||||
|
- pnl: total P&L
|
||||||
|
- trades: number of trades
|
||||||
|
- win_rate: % winners
|
||||||
|
- profit_factor: wins/losses ratio
|
||||||
|
- max_drawdown: max drawdown $
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Create v11 inputs
|
||||||
|
inputs = MoneyLineV11Inputs(
|
||||||
|
flip_threshold=config['flip_threshold'],
|
||||||
|
adx_min=config['adx_min'],
|
||||||
|
long_pos_max=config['long_pos_max'],
|
||||||
|
short_pos_min=config['short_pos_min'],
|
||||||
|
vol_min=config['vol_min'],
|
||||||
|
entry_buffer_atr=config['entry_buffer_atr'],
|
||||||
|
rsi_long_min=config['rsi_long_min'],
|
||||||
|
rsi_short_max=config['rsi_short_max'],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Generate signals
|
||||||
|
signals = money_line_v11_signals(df, inputs)
|
||||||
|
|
||||||
|
if not signals:
|
||||||
|
return {
|
||||||
|
'params': config,
|
||||||
|
'pnl': 0.0,
|
||||||
|
'trades': 0,
|
||||||
|
'win_rate': 0.0,
|
||||||
|
'profit_factor': 0.0,
|
||||||
|
'max_drawdown': 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Simple backtesting: track equity curve
|
||||||
|
equity = 1000.0 # Starting capital
|
||||||
|
peak_equity = equity
|
||||||
|
max_drawdown = 0.0
|
||||||
|
wins = 0
|
||||||
|
losses = 0
|
||||||
|
win_pnl = 0.0
|
||||||
|
loss_pnl = 0.0
|
||||||
|
|
||||||
|
for signal in signals:
|
||||||
|
# Simple trade simulation
|
||||||
|
# TP1 at +0.86%, SL at -1.29% (ATR-based defaults)
|
||||||
|
entry = signal.entry_price
|
||||||
|
|
||||||
|
# Look ahead in data to see if TP or SL hit
|
||||||
|
signal_idx = df.index.get_loc(signal.timestamp)
|
||||||
|
|
||||||
|
# Look ahead up to 100 bars
|
||||||
|
max_bars = min(100, len(df) - signal_idx - 1)
|
||||||
|
if max_bars <= 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
future_data = df.iloc[signal_idx+1:signal_idx+1+max_bars]
|
||||||
|
|
||||||
|
if signal.direction == "long":
|
||||||
|
tp_price = entry * 1.0086 # +0.86%
|
||||||
|
sl_price = entry * 0.9871 # -1.29%
|
||||||
|
|
||||||
|
# Check if TP or SL hit
|
||||||
|
hit_tp = (future_data['high'] >= tp_price).any()
|
||||||
|
hit_sl = (future_data['low'] <= sl_price).any()
|
||||||
|
|
||||||
|
if hit_tp:
|
||||||
|
pnl = 1000.0 * 0.0086 # $8.60 on $1000 position
|
||||||
|
equity += pnl
|
||||||
|
wins += 1
|
||||||
|
win_pnl += pnl
|
||||||
|
elif hit_sl:
|
||||||
|
pnl = -1000.0 * 0.0129 # -$12.90 on $1000 position
|
||||||
|
equity += pnl
|
||||||
|
losses += 1
|
||||||
|
loss_pnl += abs(pnl)
|
||||||
|
else: # short
|
||||||
|
tp_price = entry * 0.9914 # -0.86%
|
||||||
|
sl_price = entry * 1.0129 # +1.29%
|
||||||
|
|
||||||
|
# Check if TP or SL hit
|
||||||
|
hit_tp = (future_data['low'] <= tp_price).any()
|
||||||
|
hit_sl = (future_data['high'] >= sl_price).any()
|
||||||
|
|
||||||
|
if hit_tp:
|
||||||
|
pnl = 1000.0 * 0.0086 # $8.60 on $1000 position
|
||||||
|
equity += pnl
|
||||||
|
wins += 1
|
||||||
|
win_pnl += pnl
|
||||||
|
elif hit_sl:
|
||||||
|
pnl = -1000.0 * 0.0129 # -$12.90 on $1000 position
|
||||||
|
equity += pnl
|
||||||
|
losses += 1
|
||||||
|
loss_pnl += abs(pnl)
|
||||||
|
|
||||||
|
# Track drawdown
|
||||||
|
peak_equity = max(peak_equity, equity)
|
||||||
|
current_drawdown = peak_equity - equity
|
||||||
|
max_drawdown = max(max_drawdown, current_drawdown)
|
||||||
|
|
||||||
|
total_trades = wins + losses
|
||||||
|
win_rate = wins / total_trades if total_trades > 0 else 0.0
|
||||||
|
profit_factor = win_pnl / loss_pnl if loss_pnl > 0 else (float('inf') if win_pnl > 0 else 0.0)
|
||||||
|
total_pnl = equity - 1000.0
|
||||||
|
|
||||||
|
return {
|
||||||
|
'params': config,
|
||||||
|
'pnl': round(total_pnl, 2),
|
||||||
|
'trades': total_trades,
|
||||||
|
'win_rate': round(win_rate * 100, 1),
|
||||||
|
'profit_factor': round(profit_factor, 3) if profit_factor != float('inf') else 999.0,
|
||||||
|
'max_drawdown': round(max_drawdown, 2),
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"✗ Error backtesting config: {e}")
|
||||||
|
return {
|
||||||
|
'params': config,
|
||||||
|
'pnl': 0.0,
|
||||||
|
'trades': 0,
|
||||||
|
'win_rate': 0.0,
|
||||||
|
'profit_factor': 0.0,
|
||||||
|
'max_drawdown': 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_parameter_combinations() -> List[Dict[str, Any]]:
|
||||||
|
"""Generate all 256 parameter combinations"""
|
||||||
|
keys = PARAMETER_GRID.keys()
|
||||||
|
values = PARAMETER_GRID.values()
|
||||||
|
|
||||||
|
combinations = []
|
||||||
|
for combo in itertools.product(*values):
|
||||||
|
config = dict(zip(keys, combo))
|
||||||
|
combinations.append(config)
|
||||||
|
|
||||||
|
return combinations
|
||||||
|
|
||||||
|
|
||||||
|
def process_chunk(data_file: str, chunk_id: str, start_idx: int, end_idx: int):
|
||||||
|
"""Process a chunk of parameter combinations"""
|
||||||
|
print(f"\n{'='*60}")
|
||||||
|
print(f"V11 Test Worker - {chunk_id}")
|
||||||
|
print(f"Processing combinations {start_idx} to {end_idx-1}")
|
||||||
|
print(f"{'='*60}\n")
|
||||||
|
|
||||||
|
# Load market data
|
||||||
|
df = load_market_data(data_file)
|
||||||
|
|
||||||
|
# Generate all combinations
|
||||||
|
all_combos = generate_parameter_combinations()
|
||||||
|
print(f"✓ Generated {len(all_combos)} total combinations")
|
||||||
|
|
||||||
|
# Get this chunk's combinations
|
||||||
|
chunk_combos = all_combos[start_idx:end_idx]
|
||||||
|
print(f"✓ Processing {len(chunk_combos)} combinations in this chunk\n")
|
||||||
|
|
||||||
|
# Backtest with multiprocessing
|
||||||
|
print(f"⚡ Starting {MAX_WORKERS}-core backtest...\n")
|
||||||
|
|
||||||
|
with Pool(processes=MAX_WORKERS) as pool:
|
||||||
|
backtest_func = functools.partial(backtest_config, df)
|
||||||
|
results = pool.map(backtest_func, chunk_combos)
|
||||||
|
|
||||||
|
print(f"\n✓ Completed {len(results)} backtests")
|
||||||
|
|
||||||
|
# Write results to CSV
|
||||||
|
output_dir = Path('v11_test_results')
|
||||||
|
output_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
csv_file = output_dir / f"{chunk_id}_results.csv"
|
||||||
|
|
||||||
|
with open(csv_file, 'w', newline='') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
|
||||||
|
# Header
|
||||||
|
writer.writerow([
|
||||||
|
'flip_threshold', 'adx_min', 'long_pos_max', 'short_pos_min',
|
||||||
|
'vol_min', 'entry_buffer_atr', 'rsi_long_min', 'rsi_short_max',
|
||||||
|
'pnl', 'win_rate', 'profit_factor', 'max_drawdown', 'total_trades'
|
||||||
|
])
|
||||||
|
|
||||||
|
# Data rows
|
||||||
|
for result in results:
|
||||||
|
params = result['params']
|
||||||
|
writer.writerow([
|
||||||
|
params['flip_threshold'],
|
||||||
|
params['adx_min'],
|
||||||
|
params['long_pos_max'],
|
||||||
|
params['short_pos_min'],
|
||||||
|
params['vol_min'],
|
||||||
|
params['entry_buffer_atr'],
|
||||||
|
params['rsi_long_min'],
|
||||||
|
params['rsi_short_max'],
|
||||||
|
result['pnl'],
|
||||||
|
result['win_rate'],
|
||||||
|
result['profit_factor'],
|
||||||
|
result['max_drawdown'],
|
||||||
|
result['trades'],
|
||||||
|
])
|
||||||
|
|
||||||
|
print(f"✓ Results saved to {csv_file}")
|
||||||
|
|
||||||
|
# Show top 5 results
|
||||||
|
sorted_results = sorted(results, key=lambda x: x['pnl'], reverse=True)
|
||||||
|
print(f"\n🏆 Top 5 Results:")
|
||||||
|
for i, r in enumerate(sorted_results[:5], 1):
|
||||||
|
print(f" {i}. PnL: ${r['pnl']:,.2f} | Trades: {r['trades']} | WR: {r['win_rate']}%")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) != 4:
|
||||||
|
print("Usage: python v11_test_worker.py <data_file> <chunk_id> <start_idx>")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
data_file = sys.argv[1]
|
||||||
|
chunk_id = sys.argv[2]
|
||||||
|
start_idx = int(sys.argv[3])
|
||||||
|
|
||||||
|
# Calculate end index (128 combos per chunk)
|
||||||
|
end_idx = start_idx + 128
|
||||||
|
|
||||||
|
process_chunk(data_file, chunk_id, start_idx, end_idx)
|
||||||
Reference in New Issue
Block a user