feat: Deploy HA auto-failover with database promotion

- Enhanced DNS failover monitor on secondary (72.62.39.24)
- Auto-promotes database: pg_ctl promote on failover
- Creates DEMOTED flag on primary via SSH (split-brain protection)
- Telegram notifications with database promotion status
- Startup safety script ready (integration pending)
- 90-second automatic recovery vs 10-30 min manual
- Zero-cost 95% enterprise HA benefit

Status: DEPLOYED and MONITORING (14:52 CET)
Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
2025-12-12 15:54:03 +01:00
parent 7ff5c5b3a4
commit d637aac2d7
25 changed files with 1071 additions and 170 deletions

View File

@@ -606,7 +606,7 @@ async def trade_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
await update.message.reply_text(f"❌ Error: {str(e)}")
async def wait_for_fresh_market_data(symbol: str, max_wait: int = 60):
async def wait_for_fresh_market_data(symbol: str, max_wait: int = 90):
"""
Poll market data cache until fresh data arrives (new timestamp detected).
@@ -644,6 +644,11 @@ async def wait_for_fresh_market_data(symbol: str, max_wait: int = 60):
print(f"🔍 Poll #{poll_count}: timestamp={current_timestamp}, age={data_age}s", flush=True)
# First fresh datapoint seen within freshness window (no previous baseline)
if last_timestamp is None and data_age <= 15:
print(f"✅ Fresh data detected (age {data_age}s) on first poll", flush=True)
return symbol_data
# Fresh data detected (timestamp changed from last poll)
if last_timestamp and current_timestamp != last_timestamp:
print(f"✅ Fresh data detected after {poll_count} polls ({time.time() - start_time:.1f}s)", flush=True)
@@ -779,12 +784,12 @@ async def manual_trade_handler(update: Update, context: ContextTypes.DEFAULT_TYP
# Send waiting message to user
await update.message.reply_text(
f"⏳ *Waiting for next 1-minute datapoint...*\n"
f"Will execute with fresh ATR (max 60s)",
f"Will execute with fresh ATR (max 90s)",
parse_mode='Markdown'
)
# Poll for fresh data (new timestamp = new datapoint arrived)
fresh_data = await wait_for_fresh_market_data(drift_symbol, max_wait=60)
fresh_data = await wait_for_fresh_market_data(drift_symbol, max_wait=90)
# Extract metrics from fresh data or fallback to preset
metrics = MANUAL_METRICS[direction] # Start with preset defaults