root/trading_bot_v4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Deploy HA auto-failover with database promotion

Browse Source 
- Enhanced DNS failover monitor on secondary (72.62.39.24)
- Auto-promotes database: pg_ctl promote on failover
- Creates DEMOTED flag on primary via SSH (split-brain protection)
- Telegram notifications with database promotion status
- Startup safety script ready (integration pending)
- 90-second automatic recovery vs 10-30 min manual
- Zero-cost 95% enterprise HA benefit

Status: DEPLOYED and MONITORING (14:52 CET)
Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
2025-12-12 15:54:03 +01:00
parent 7ff5c5b3a4
commit d637aac2d7
25 changed files with 1071 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
lib/drift/orders.ts
View File

@@ -54,6 +54,8 @@ export interface PlaceExitOrdersResult {
success: boolean
signatures?: string[]
error?: string
expectedOrders?: number
placedOrders?: number
}
export interface PlaceExitOrdersOptions {
@@ -271,6 +273,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
}
const signatures: string[] = []
let expectedOrders = 0
// Helper to compute base asset amount from USD notional and price
// CRITICAL FIX (Dec 10, 2025): Must use SPECIFIC PRICE for each order (TP1 price, TP2 price, SL price)
@@ -285,9 +288,16 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
// CRITICAL FIX: TP2 must be percentage of REMAINING size after TP1, not original size
const tp1USD = (options.positionSizeUSD * options.tp1SizePercent) / 100
const remainingAfterTP1 = options.positionSizeUSD - tp1USD
const requestedTp2Percent = options.tp2SizePercent ?? 100
const normalizedTp2Percent = requestedTp2Percent > 0 ? requestedTp2Percent : 100
const requestedTp2Percent = options.tp2SizePercent
// Allow explicit 0% to mean "no TP2 order" without forcing it back to 100%
const normalizedTp2Percent = requestedTp2Percent === undefined
? 100
: Math.max(0, requestedTp2Percent)
const tp2USD = (remainingAfterTP1 * normalizedTp2Percent) / 100
if (normalizedTp2Percent === 0) {
logger.log('ℹ️ TP2 on-chain order skipped (trigger-only; software handles trailing)')
}
logger.log(`📊 Exit order sizes:`)
logger.log(` TP1: ${options.tp1SizePercent}% of $${options.positionSizeUSD.toFixed(2)} = $${tp1USD.toFixed(2)}`)
@@ -302,6 +312,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
if (tp1USD > 0) {
const baseAmount = usdToBase(tp1USD, options.tp1Price) // Use TP1 price
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
expectedOrders += 1
const orderParams: any = {
orderType: OrderType.LIMIT,
marketIndex: marketConfig.driftMarketIndex,
@@ -326,6 +337,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
if (tp2USD > 0) {
const baseAmount = usdToBase(tp2USD, options.tp2Price) // Use TP2 price
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
expectedOrders += 1
const orderParams: any = {
orderType: OrderType.LIMIT,
marketIndex: marketConfig.driftMarketIndex,
@@ -355,17 +367,22 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
const slUSD = options.positionSizeUSD
const slBaseAmount = usdToBase(slUSD, options.stopLossPrice) // Use SL price
// Calculate expected number of orders for validation (Bug #76 fix)
const useDualStops = options.useDualStops ?? false
const expectedOrderCount = 2 + (useDualStops ? 2 : 1) // TP1 + TP2 + (soft+hard SL OR single SL)
logger.log(`📊 Expected exit orders: TP1/TP2 that meet min size + ${useDualStops ? 'soft+hard SL' : 'single SL'}`)
logger.log(`📊 Expected ${expectedOrderCount} exit orders total (TP1 + TP2 + ${useDualStops ? 'dual stops' : 'single stop'})`)
const minOrderLamports = Math.floor(marketConfig.minOrderSize * 1e9)
if (slBaseAmount < minOrderLamports) {
const errorMsg = `Stop loss size below market minimum (base ${slBaseAmount} < min ${minOrderLamports})`
console.error(`❌ ${errorMsg}`)
return { success: false, error: errorMsg, signatures, expectedOrders, placedOrders: signatures.length }
}
if (slBaseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
if (slBaseAmount >= minOrderLamports) {
if (useDualStops && options.softStopPrice && options.hardStopPrice) {
// ============== DUAL STOP SYSTEM ==============
logger.log('🛡️🛡️ Placing DUAL STOP SYSTEM...')
expectedOrders += 2
try {
// 1. Soft Stop (TRIGGER_LIMIT) - Avoids wicks
@@ -438,6 +455,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
// ============== SINGLE STOP SYSTEM ==============
const useStopLimit = options.useStopLimit ?? false
const stopLimitBuffer = options.stopLimitBuffer ?? 0.5
expectedOrders += 1
try {
if (useStopLimit) {
@@ -500,26 +518,26 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
throw new Error(`Stop loss placement failed: ${slError instanceof Error ? slError.message : 'Unknown error'}`)
}
}
} else {
logger.log('⚠️ SL size below market min, skipping on-chain SL')
}
// CRITICAL VALIDATION (Bug #76 fix): Verify all expected orders were placed
if (signatures.length < expectedOrderCount) {
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrderCount}, got ${signatures.length}. Position is UNPROTECTED!`
const placedOrders = signatures.length
if (placedOrders < expectedOrders) {
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrders}, got ${placedOrders}. Position is UNPROTECTED!`
console.error(`❌ ${errorMsg}`)
console.error(` Expected: TP1 + TP2 + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
console.error(` Got ${signatures.length} signatures:`, signatures)
console.error(` Expected: TP1/TP2 that met min + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
console.error(` Got ${placedOrders} signatures:`, signatures)
return {
success: false,
error: errorMsg,
signatures // Return partial signatures for debugging
signatures, // Return partial signatures for debugging
expectedOrders,
placedOrders,
}
}
logger.log(`✅ All ${expectedOrderCount} exit orders placed successfully`)
return { success: true, signatures }
logger.log(`✅ All ${expectedOrders} exit orders placed successfully`)
return { success: true, signatures, expectedOrders, placedOrders }
} catch (error) {
console.error('❌ Failed to place exit orders:', error)
return { success: false, error: error instanceof Error ? error.message : 'Unknown error' }
@@ -533,20 +551,20 @@ export async function closePosition(
params: ClosePositionParams
): Promise<ClosePositionResult> {
try {
logger.log('📊 Closing position:', params)
const driftService = getDriftService()
const marketConfig = getMarketConfig(params.symbol)
const driftService = await getDriftService()
const driftClient = driftService.getClient()
// Get current position
const position = await driftService.getPosition(marketConfig.driftMarketIndex)
if (!position || position.side === 'none') {
throw new Error(`No active position for ${params.symbol}`)
console.warn(`⚠️ No open position found for ${params.symbol}, skipping close request`)
return {
success: false,
error: 'No open position to close',
}
}
console.log(`🔍 CLOSE POSITION DEBUG:`)
logger.log('📊 Closing position:', params)
console.log(` params.percentToClose: ${params.percentToClose}`)
console.log(` position.size: ${position.size}`)
console.log(` marketConfig.minOrderSize: ${marketConfig.minOrderSize}`)
@@ -627,6 +645,7 @@ export async function closePosition(
// BUT: Use timeout to prevent API hangs during network congestion
logger.log('⏳ Confirming transaction on-chain (30s timeout)...')
const connection = driftService.getTradeConnection() // Use Alchemy for trade operations
let confirmationTimedOut = false
try {
const confirmationPromise = connection.confirmTransaction(txSig, 'confirmed')
@@ -644,10 +663,11 @@ export async function closePosition(
logger.log('✅ Transaction confirmed on-chain')
} catch (timeoutError: any) {
if (timeoutError.message === 'Transaction confirmation timeout') {
confirmationTimedOut = true
console.warn('⚠️ Transaction confirmation timed out after 30s')
console.warn(' Order may still execute - check Drift UI')
console.warn(` Transaction signature: ${txSig}`)
// Continue anyway - order was submitted and will likely execute
// Continue but flag for Position Manager verification so we do not drop tracking
} else {
throw timeoutError
}
@@ -726,6 +746,7 @@ export async function closePosition(
closePrice: oraclePrice,
closedSize: sizeToClose,
realizedPnL,
needsVerification: confirmationTimedOut, // Keep monitoring if confirmation never arrived
}
} catch (error) {

16
lib/health/position-manager-health.ts
View File

@@ -51,8 +51,8 @@ export async function checkPositionManagerHealth(): Promise<HealthCheckResult> {
// Get Position Manager state
const pm = await getInitializedPositionManager()
const pmState = (pm as any)
const pmActiveTrades = pmState.activeTrades?.size || 0
const pmMonitoring = pmState.isMonitoring || false
let pmActiveTrades = pmState.activeTrades?.size || 0
let pmMonitoring = pmState.isMonitoring || false
// Get Drift positions
const driftService = getDriftService()
@@ -60,6 +60,18 @@ export async function checkPositionManagerHealth(): Promise<HealthCheckResult> {
const driftPositions = positions.filter(p => Math.abs(p.size) > 0).length
// CRITICAL CHECK #1: DB has open trades but PM not monitoring
if (dbOpenCount > 0 && !pmMonitoring) {
console.log('🛠️ Health monitor: Attempting automatic monitoring restore from DB...')
try {
await pm.initialize(true)
pmActiveTrades = (pm as any).activeTrades?.size || 0
pmMonitoring = (pm as any).isMonitoring || false
} catch (restoreError) {
console.error('❌ Failed to auto-restore monitoring:', restoreError)
}
}
// Re-check after attempted restore
if (dbOpenCount > 0 && !pmMonitoring) {
issues.push(`❌ CRITICAL: ${dbOpenCount} open trades in DB but Position Manager NOT monitoring!`)
issues.push(` This means NO TP/SL protection, NO monitoring, UNCONTROLLED RISK`)

5
lib/notifications/telegram.ts
View File

@@ -121,8 +121,9 @@ ${directionEmoji} ${options.symbol} ${options.direction.toUpperCase()}
📍 Price: $${options.originalPrice.toFixed(2)}
🧠 Watching for price confirmation...
✅ Will enter if ${options.direction === 'long' ? '+0.3%' : '-0.3%'}
❌ Will abandon if ${options.direction === 'long' ? '-0.4%' : '+0.4%'}`
✅ Will enter if ${options.direction === 'long' ? '+0.15%' : '-0.15%'}
❌ Will abandon if ${options.direction === 'long' ? '-1.0%' : '+1.0%'}
`
break
case 'confirmed':

6
lib/trading/market-data-cache.ts
View File

@@ -48,18 +48,18 @@ class MarketDataCache {
const data = this.cache.get(symbol)
if (!data) {
logger.log(`⚠️ No cached data for ${symbol}`)
console.log(`⚠️ No cached data for ${symbol}`)
return null
}
const ageSeconds = Math.round((Date.now() - data.timestamp) / 1000)
if (Date.now() - data.timestamp > this.MAX_AGE_MS) {
logger.log(`⏰ Cached data for ${symbol} is stale (${ageSeconds}s old, max 300s)`)
console.log(`⏰ Cached data for ${symbol} is stale (${ageSeconds}s old, max 300s)`)
return null
}
logger.log(`✅ Using fresh TradingView data for ${symbol} (${ageSeconds}s old)`)
console.log(`✅ Using fresh TradingView data for ${symbol} (${ageSeconds}s old)`)
return data
}

14
lib/trading/position-manager.ts
View File

@@ -99,11 +99,17 @@ export class PositionManager {
/**
* Initialize and restore active trades from database
*/
async initialize(): Promise<void> {
if (this.initialized) {
async initialize(forceReload: boolean = false): Promise<void> {
if (this.initialized && !forceReload) {
return
}
if (forceReload) {
logger.log('🔄 Force reloading Position Manager state from database')
this.activeTrades.clear()
this.isMonitoring = false
}
logger.log('🔄 Restoring active trades from database...')
try {
@@ -2069,7 +2075,9 @@ export class PositionManager {
lastPrice: trade.lastPrice,
})
} catch (error) {
console.error('❌ Failed to save trade state:', error)
const tradeId = (trade as any).id ?? 'unknown'
const positionId = trade.positionId ?? 'unknown'
console.error(`❌ Failed to save trade state (tradeId=${tradeId}, positionId=${positionId}, symbol=${trade.symbol}):`, error)
// Don't throw - state save is non-critical
}
}

8
lib/trading/smart-entry-timer.ts
View File

@@ -475,6 +475,10 @@ export class SmartEntryTimer {
const stopLossPrice = this.calculatePrice(fillPrice, slPercent, signal.direction)
const tp1Price = this.calculatePrice(fillPrice, tp1Percent, signal.direction)
const tp2Price = this.calculatePrice(fillPrice, tp2Percent, signal.direction)
const effectiveTp2SizePercent =
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0)
// Dual stops if enabled
let softStopPrice: number | undefined
@@ -496,7 +500,7 @@ export class SmartEntryTimer {
tp2Price,
stopLossPrice,
tp1SizePercent: config.takeProfit1SizePercent ?? 75,
tp2SizePercent: config.takeProfit2SizePercent ?? 0,
tp2SizePercent: effectiveTp2SizePercent,
direction: signal.direction,
useDualStops: config.useDualStops,
softStopPrice,
@@ -525,7 +529,7 @@ export class SmartEntryTimer {
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent,
tp2SizePercent: config.takeProfit2SizePercent,
tp2SizePercent: effectiveTp2SizePercent,
entryOrderTx: openResult.transactionSignature,
atrAtEntry: signal.originalSignalData.atr,
adxAtEntry: signal.originalSignalData.adx,

54
lib/trading/smart-validation-queue.ts
View File

@@ -103,8 +103,8 @@ class SmartValidationQueue {
qualityScore: params.qualityScore,
blockedAt: Date.now(),
entryWindowMinutes: 90, // Two-stage: watch for 90 minutes
confirmationThreshold: 0.15, // Two-stage: need +0.15% move to confirm
maxDrawdown: -0.4, // Abandon if -0.4% against direction (unchanged)
confirmationThreshold: 0.3, // Two-stage: need +0.3% move to confirm
maxDrawdown: -1.0, // Abandon if -1.0% against direction (widened from 0.4%)
highestPrice: params.originalPrice,
lowestPrice: params.originalPrice,
status: 'pending',
@@ -211,17 +211,46 @@ class SmartValidationQueue {
return
}
// Get current price from market data cache
const marketDataCache = getMarketDataCache()
const cachedData = marketDataCache.get(signal.symbol)
// CRITICAL FIX (Dec 11, 2025): Query database for latest 1-minute data instead of cache
// Cache singleton issue: API routes and validation queue have separate instances
// Database is single source of truth for market data
let currentPrice: number
let priceDataAge: number
try {
const { getPrismaClient } = await import('../database/trades')
const prisma = getPrismaClient()
// Get most recent market data within last 2 minutes
const recentData = await prisma.marketData.findFirst({
where: {
symbol: signal.symbol,
timestamp: {
gte: new Date(Date.now() - 2 * 60 * 1000) // Last 2 minutes
}
},
orderBy: {
timestamp: 'desc'
}
})
if (!cachedData || !cachedData.currentPrice) {
logger.log(`⚠️ No price data for ${signal.symbol}, skipping validation`)
if (!recentData) {
console.log(`⚠️ No recent market data for ${signal.symbol} in database (last 2 min), skipping validation`)
return
}
currentPrice = recentData.price
priceDataAge = Math.round((Date.now() - recentData.timestamp.getTime()) / 1000)
console.log(`✅ Using database market data for ${signal.symbol} (${priceDataAge}s old, price: $${currentPrice.toFixed(2)})`)
} catch (dbError) {
console.error(`❌ Database query failed for ${signal.symbol}:`, dbError)
return
}
const currentPrice = cachedData.currentPrice
const priceChange = ((currentPrice - signal.originalPrice) / signal.originalPrice) * 100
console.log(`📊 ${signal.symbol} ${signal.direction.toUpperCase()}: Original $${signal.originalPrice.toFixed(2)} → Current $${currentPrice.toFixed(2)} = ${priceChange >= 0 ? '+' : ''}${priceChange.toFixed(2)}%`)
// Update price extremes
if (!signal.highestPrice || currentPrice > signal.highestPrice) {
@@ -468,9 +497,8 @@ export async function startSmartValidation(): Promise<void> {
const recentBlocked = await prisma.blockedSignal.findMany({
where: {
blockReason: 'QUALITY_SCORE_TOO_LOW',
signalQualityScore: { gte: 50, lt: 90 }, // Marginal quality range
createdAt: { gte: ninetyMinutesAgo },
blockReason: 'SMART_VALIDATION_QUEUED', // FIXED Dec 12, 2025: Look for queued signals only
createdAt: { gte: ninetyMinutesAgo }, // Match entry window (90 minutes)
},
orderBy: { createdAt: 'desc' },
})
@@ -480,10 +508,10 @@ export async function startSmartValidation(): Promise<void> {
// Re-queue each signal
for (const signal of recentBlocked) {
await queue.addSignal({
blockReason: 'QUALITY_SCORE_TOO_LOW',
blockReason: 'SMART_VALIDATION_QUEUED',
symbol: signal.symbol,
direction: signal.direction as 'long' | 'short',
originalPrice: signal.entryPrice,
originalPrice: signal.signalPrice,
qualityScore: signal.signalQualityScore || 0,
atr: signal.atr || undefined,
adx: signal.adx || undefined,