feat: Deploy HA auto-failover with database promotion
- Enhanced DNS failover monitor on secondary (72.62.39.24) - Auto-promotes database: pg_ctl promote on failover - Creates DEMOTED flag on primary via SSH (split-brain protection) - Telegram notifications with database promotion status - Startup safety script ready (integration pending) - 90-second automatic recovery vs 10-30 min manual - Zero-cost 95% enterprise HA benefit Status: DEPLOYED and MONITORING (14:52 CET) Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
@@ -48,18 +48,18 @@ class MarketDataCache {
|
||||
const data = this.cache.get(symbol)
|
||||
|
||||
if (!data) {
|
||||
logger.log(`⚠️ No cached data for ${symbol}`)
|
||||
console.log(`⚠️ No cached data for ${symbol}`)
|
||||
return null
|
||||
}
|
||||
|
||||
const ageSeconds = Math.round((Date.now() - data.timestamp) / 1000)
|
||||
|
||||
if (Date.now() - data.timestamp > this.MAX_AGE_MS) {
|
||||
logger.log(`⏰ Cached data for ${symbol} is stale (${ageSeconds}s old, max 300s)`)
|
||||
console.log(`⏰ Cached data for ${symbol} is stale (${ageSeconds}s old, max 300s)`)
|
||||
return null
|
||||
}
|
||||
|
||||
logger.log(`✅ Using fresh TradingView data for ${symbol} (${ageSeconds}s old)`)
|
||||
console.log(`✅ Using fresh TradingView data for ${symbol} (${ageSeconds}s old)`)
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
@@ -99,11 +99,17 @@ export class PositionManager {
|
||||
/**
|
||||
* Initialize and restore active trades from database
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
async initialize(forceReload: boolean = false): Promise<void> {
|
||||
if (this.initialized && !forceReload) {
|
||||
return
|
||||
}
|
||||
|
||||
if (forceReload) {
|
||||
logger.log('🔄 Force reloading Position Manager state from database')
|
||||
this.activeTrades.clear()
|
||||
this.isMonitoring = false
|
||||
}
|
||||
|
||||
logger.log('🔄 Restoring active trades from database...')
|
||||
|
||||
try {
|
||||
@@ -2069,7 +2075,9 @@ export class PositionManager {
|
||||
lastPrice: trade.lastPrice,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to save trade state:', error)
|
||||
const tradeId = (trade as any).id ?? 'unknown'
|
||||
const positionId = trade.positionId ?? 'unknown'
|
||||
console.error(`❌ Failed to save trade state (tradeId=${tradeId}, positionId=${positionId}, symbol=${trade.symbol}):`, error)
|
||||
// Don't throw - state save is non-critical
|
||||
}
|
||||
}
|
||||
|
||||
@@ -475,6 +475,10 @@ export class SmartEntryTimer {
|
||||
const stopLossPrice = this.calculatePrice(fillPrice, slPercent, signal.direction)
|
||||
const tp1Price = this.calculatePrice(fillPrice, tp1Percent, signal.direction)
|
||||
const tp2Price = this.calculatePrice(fillPrice, tp2Percent, signal.direction)
|
||||
const effectiveTp2SizePercent =
|
||||
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
|
||||
? 0
|
||||
: (config.takeProfit2SizePercent ?? 0)
|
||||
|
||||
// Dual stops if enabled
|
||||
let softStopPrice: number | undefined
|
||||
@@ -496,7 +500,7 @@ export class SmartEntryTimer {
|
||||
tp2Price,
|
||||
stopLossPrice,
|
||||
tp1SizePercent: config.takeProfit1SizePercent ?? 75,
|
||||
tp2SizePercent: config.takeProfit2SizePercent ?? 0,
|
||||
tp2SizePercent: effectiveTp2SizePercent,
|
||||
direction: signal.direction,
|
||||
useDualStops: config.useDualStops,
|
||||
softStopPrice,
|
||||
@@ -525,7 +529,7 @@ export class SmartEntryTimer {
|
||||
takeProfit1Price: tp1Price,
|
||||
takeProfit2Price: tp2Price,
|
||||
tp1SizePercent: config.takeProfit1SizePercent,
|
||||
tp2SizePercent: config.takeProfit2SizePercent,
|
||||
tp2SizePercent: effectiveTp2SizePercent,
|
||||
entryOrderTx: openResult.transactionSignature,
|
||||
atrAtEntry: signal.originalSignalData.atr,
|
||||
adxAtEntry: signal.originalSignalData.adx,
|
||||
|
||||
@@ -103,8 +103,8 @@ class SmartValidationQueue {
|
||||
qualityScore: params.qualityScore,
|
||||
blockedAt: Date.now(),
|
||||
entryWindowMinutes: 90, // Two-stage: watch for 90 minutes
|
||||
confirmationThreshold: 0.15, // Two-stage: need +0.15% move to confirm
|
||||
maxDrawdown: -0.4, // Abandon if -0.4% against direction (unchanged)
|
||||
confirmationThreshold: 0.3, // Two-stage: need +0.3% move to confirm
|
||||
maxDrawdown: -1.0, // Abandon if -1.0% against direction (widened from 0.4%)
|
||||
highestPrice: params.originalPrice,
|
||||
lowestPrice: params.originalPrice,
|
||||
status: 'pending',
|
||||
@@ -211,17 +211,46 @@ class SmartValidationQueue {
|
||||
return
|
||||
}
|
||||
|
||||
// Get current price from market data cache
|
||||
const marketDataCache = getMarketDataCache()
|
||||
const cachedData = marketDataCache.get(signal.symbol)
|
||||
// CRITICAL FIX (Dec 11, 2025): Query database for latest 1-minute data instead of cache
|
||||
// Cache singleton issue: API routes and validation queue have separate instances
|
||||
// Database is single source of truth for market data
|
||||
let currentPrice: number
|
||||
let priceDataAge: number
|
||||
|
||||
try {
|
||||
const { getPrismaClient } = await import('../database/trades')
|
||||
const prisma = getPrismaClient()
|
||||
|
||||
// Get most recent market data within last 2 minutes
|
||||
const recentData = await prisma.marketData.findFirst({
|
||||
where: {
|
||||
symbol: signal.symbol,
|
||||
timestamp: {
|
||||
gte: new Date(Date.now() - 2 * 60 * 1000) // Last 2 minutes
|
||||
}
|
||||
},
|
||||
orderBy: {
|
||||
timestamp: 'desc'
|
||||
}
|
||||
})
|
||||
|
||||
if (!cachedData || !cachedData.currentPrice) {
|
||||
logger.log(`⚠️ No price data for ${signal.symbol}, skipping validation`)
|
||||
if (!recentData) {
|
||||
console.log(`⚠️ No recent market data for ${signal.symbol} in database (last 2 min), skipping validation`)
|
||||
return
|
||||
}
|
||||
|
||||
currentPrice = recentData.price
|
||||
priceDataAge = Math.round((Date.now() - recentData.timestamp.getTime()) / 1000)
|
||||
|
||||
console.log(`✅ Using database market data for ${signal.symbol} (${priceDataAge}s old, price: $${currentPrice.toFixed(2)})`)
|
||||
} catch (dbError) {
|
||||
console.error(`❌ Database query failed for ${signal.symbol}:`, dbError)
|
||||
return
|
||||
}
|
||||
|
||||
const currentPrice = cachedData.currentPrice
|
||||
const priceChange = ((currentPrice - signal.originalPrice) / signal.originalPrice) * 100
|
||||
|
||||
console.log(`📊 ${signal.symbol} ${signal.direction.toUpperCase()}: Original $${signal.originalPrice.toFixed(2)} → Current $${currentPrice.toFixed(2)} = ${priceChange >= 0 ? '+' : ''}${priceChange.toFixed(2)}%`)
|
||||
|
||||
// Update price extremes
|
||||
if (!signal.highestPrice || currentPrice > signal.highestPrice) {
|
||||
@@ -468,9 +497,8 @@ export async function startSmartValidation(): Promise<void> {
|
||||
|
||||
const recentBlocked = await prisma.blockedSignal.findMany({
|
||||
where: {
|
||||
blockReason: 'QUALITY_SCORE_TOO_LOW',
|
||||
signalQualityScore: { gte: 50, lt: 90 }, // Marginal quality range
|
||||
createdAt: { gte: ninetyMinutesAgo },
|
||||
blockReason: 'SMART_VALIDATION_QUEUED', // FIXED Dec 12, 2025: Look for queued signals only
|
||||
createdAt: { gte: ninetyMinutesAgo }, // Match entry window (90 minutes)
|
||||
},
|
||||
orderBy: { createdAt: 'desc' },
|
||||
})
|
||||
@@ -480,10 +508,10 @@ export async function startSmartValidation(): Promise<void> {
|
||||
// Re-queue each signal
|
||||
for (const signal of recentBlocked) {
|
||||
await queue.addSignal({
|
||||
blockReason: 'QUALITY_SCORE_TOO_LOW',
|
||||
blockReason: 'SMART_VALIDATION_QUEUED',
|
||||
symbol: signal.symbol,
|
||||
direction: signal.direction as 'long' | 'short',
|
||||
originalPrice: signal.entryPrice,
|
||||
originalPrice: signal.signalPrice,
|
||||
qualityScore: signal.signalQualityScore || 0,
|
||||
atr: signal.atr || undefined,
|
||||
adx: signal.adx || undefined,
|
||||
|
||||
Reference in New Issue
Block a user