root/trading_bot_v4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Deploy HA auto-failover with database promotion

Browse Source 
- Enhanced DNS failover monitor on secondary (72.62.39.24)
- Auto-promotes database: pg_ctl promote on failover
- Creates DEMOTED flag on primary via SSH (split-brain protection)
- Telegram notifications with database promotion status
- Startup safety script ready (integration pending)
- 90-second automatic recovery vs 10-30 min manual
- Zero-cost 95% enterprise HA benefit

Status: DEPLOYED and MONITORING (14:52 CET)
Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
2025-12-12 15:54:03 +01:00
parent 7ff5c5b3a4
commit d637aac2d7
25 changed files with 1071 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
app/api/trading/check-risk/route.ts
View File

@@ -408,8 +408,19 @@ export async function POST(request: NextRequest): Promise<NextResponse<RiskCheck
// Get current price for the blocked signal record
const currentPrice = await getCurrentPrice(body.symbol, fallbackPrice)
// CRITICAL FIX (Dec 12, 2025): Smart validation integration
// Check if signal quality is in validation range (50-89)
const isInValidationRange = qualityScore.score >= 50 && qualityScore.score < 90
// Save blocked signal to database for future analysis
if (currentPrice > 0) {
// SMART VALIDATION QUEUE (Nov 30, 2025 - FIXED Dec 12, 2025)
// Queue marginal quality signals (50-89) for validation instead of hard-blocking
const blockReason = isInValidationRange ? 'SMART_VALIDATION_QUEUED' : 'QUALITY_SCORE_TOO_LOW'
const blockDetails = isInValidationRange
? `Score: ${qualityScore.score}/${minQualityScore} - Queued for validation (will enter if +0.3%, abandon if -1.0%)`
: `Score: ${qualityScore.score}/${minQualityScore} - ${qualityScore.reasons.join(', ')}`
await createBlockedSignal({
symbol: body.symbol,
direction: body.direction,
@@ -421,41 +432,41 @@ export async function POST(request: NextRequest): Promise<NextResponse<RiskCheck
volumeRatio: body.volumeRatio,
pricePosition: body.pricePosition,
signalQualityScore: qualityScore.score,
signalQualityVersion: 'v4', // Update this when scoring logic changes
signalQualityVersion: 'v4',
scoreBreakdown: { reasons: qualityScore.reasons },
minScoreRequired: minQualityScore, // Use direction-specific threshold
blockReason: 'QUALITY_SCORE_TOO_LOW',
blockDetails: `Score: ${qualityScore.score}/${minQualityScore} - ${qualityScore.reasons.join(', ')}`,
minScoreRequired: minQualityScore,
blockReason: blockReason,
blockDetails: blockDetails,
indicatorVersion: body.indicatorVersion || 'v5',
})
// SMART VALIDATION QUEUE (Nov 30, 2025)
// Queue marginal quality signals (50-89) for validation instead of hard-blocking
const validationQueue = getSmartValidationQueue()
// CRITICAL FIX (Dec 1, 2025): Normalize TradingView symbol format to Drift format
// Bug: Market data cache uses "SOL-PERP" but TradingView sends "SOLUSDT"
// Without normalization, validation queue can't find matching price data
// Result: Wrong/stale price shown in Telegram abandonment notifications
const normalizedSymbol = normalizeTradingViewSymbol(body.symbol)
const queued = await validationQueue.addSignal({
blockReason: 'QUALITY_SCORE_TOO_LOW',
symbol: normalizedSymbol, // Use normalized format for cache lookup
direction: body.direction,
originalPrice: currentPrice,
qualityScore: qualityScore.score,
atr: body.atr,
adx: body.adx,
rsi: body.rsi,
volumeRatio: body.volumeRatio,
pricePosition: body.pricePosition,
indicatorVersion: body.indicatorVersion || 'v5',
timeframe: body.timeframe || '5',
})
if (queued) {
console.log(`🧠 Signal queued for smart validation: ${normalizedSymbol} ${body.direction} (quality ${qualityScore.score})`)
// Add to validation queue if in range
if (isInValidationRange) {
const validationQueue = getSmartValidationQueue()
// CRITICAL FIX (Dec 1, 2025): Normalize TradingView symbol format to Drift format
const normalizedSymbol = normalizeTradingViewSymbol(body.symbol)
const queued = await validationQueue.addSignal({
blockReason: 'SMART_VALIDATION_QUEUED',
symbol: normalizedSymbol,
direction: body.direction,
originalPrice: currentPrice,
qualityScore: qualityScore.score,
atr: body.atr,
adx: body.adx,
rsi: body.rsi,
volumeRatio: body.volumeRatio,
pricePosition: body.pricePosition,
indicatorVersion: body.indicatorVersion || 'v5',
timeframe: body.timeframe || '5',
})
if (queued) {
console.log(`🧠 Signal queued for smart validation: ${normalizedSymbol} ${body.direction} (quality ${qualityScore.score})`)
}
} else {
console.log(`❌ Signal quality too low for validation: ${qualityScore.score} (need 50-89 range)`)
}
} else {
console.warn('⚠️ Skipping blocked signal save: price unavailable (quality block)')

16
app/api/trading/execute/route.ts
View File

@@ -900,6 +900,8 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
// CRITICAL FIX: Place on-chain TP/SL orders BEFORE adding to Position Manager
// This prevents race condition where Position Manager detects "external closure"
// while orders are still being placed, leaving orphaned stop loss orders
// TP2 is a software trigger only – do not place on-chain TP2 orders so the runner remains intact
const effectiveTp2SizePercent = 0
let exitOrderSignatures: string[] = []
try {
console.log('🔍 DEBUG: About to call placeExitOrders()...')
@@ -921,7 +923,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
tp2Price,
stopLossPrice,
tp1SizePercent: config.takeProfit1SizePercent ?? 75,
tp2SizePercent: config.takeProfit2SizePercent ?? 0, // 0 = activate trailing stop, don't close
tp2SizePercent: effectiveTp2SizePercent, // Always trigger-only: trailing activation only
direction: body.direction,
// Dual stop parameters
useDualStops: config.useDualStops,
@@ -950,7 +952,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
exitOrderSignatures = exitRes.signatures || []
// BUG #76 FIX: Validate expected signature count
const expectedCount = config.useDualStops ? 4 : 3 // TP1 + TP2 + (soft+hard OR single SL)
const expectedCount = exitRes.expectedOrders ?? (config.useDualStops ? 3 : 2)
if (exitOrderSignatures.length < expectedCount) {
console.error(`❌ CRITICAL: Missing exit orders!`)
console.error(` Expected: ${expectedCount} signatures (TP1 + TP2 + ${config.useDualStops ? 'Soft SL + Hard SL' : 'SL'})`)
@@ -1025,14 +1027,14 @@ export async function POST(request: NextRequest): Promise<NextResponse<ExecuteTr
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent ?? 75,
tp2SizePercent: config.takeProfit2SizePercent ?? 0, // Use ?? to allow 0 for runner system
tp2SizePercent: effectiveTp2SizePercent, // Use ?? to allow 0 for runner system
configSnapshot: config,
entryOrderTx: openResult.transactionSignature!,
tp1OrderTx: exitOrderSignatures[0],
tp2OrderTx: exitOrderSignatures[1],
slOrderTx: config.useDualStops ? undefined : exitOrderSignatures[2],
softStopOrderTx: config.useDualStops ? exitOrderSignatures[2] : undefined,
hardStopOrderTx: config.useDualStops ? exitOrderSignatures[3] : undefined,
tp2OrderTx: undefined, // TP2 is software-only trigger; no on-chain TP2 order
slOrderTx: config.useDualStops ? undefined : exitOrderSignatures[1],
softStopOrderTx: config.useDualStops ? exitOrderSignatures[1] : undefined,
hardStopOrderTx: config.useDualStops ? exitOrderSignatures[2] : undefined,
softStopPrice,
hardStopPrice,
signalSource: body.timeframe === 'manual' ? 'manual' : 'tradingview', // Identify manual Telegram trades

15
app/api/trading/market-data/route.ts
View File

@@ -37,12 +37,15 @@ function normalizeTradingViewSymbol(tvSymbol: string): string {
'FARTUSDT': 'FARTCOIN-PERP',
'FART': 'FARTCOIN-PERP',
'SOLUSDT': 'SOL-PERP',
'SOLUSDT.P': 'SOL-PERP',
'SOLUSD': 'SOL-PERP',
'SOL': 'SOL-PERP',
'ETHUSDT': 'ETH-PERP',
'ETHUSDT.P': 'ETH-PERP',
'ETHUSD': 'ETH-PERP',
'ETH': 'ETH-PERP',
'BTCUSDT': 'BTC-PERP',
'BTCUSDT.P': 'BTC-PERP',
'BTCUSD': 'BTC-PERP',
'BTC': 'BTC-PERP'
}
@@ -80,6 +83,16 @@ export async function POST(request: NextRequest) {
}
const driftSymbol = normalizeTradingViewSymbol(body.symbol)
// Parse timestamp defensively – fall back to now if malformed to avoid dropping data
const parsedTimestamp = body.timestamp ? new Date(body.timestamp) : new Date()
const timestamp = Number.isNaN(parsedTimestamp.getTime()) ? new Date() : parsedTimestamp
if (body.timestamp && Number.isNaN(parsedTimestamp.getTime())) {
console.warn('⚠️ Invalid timestamp in market data payload, falling back to now', {
symbol: driftSymbol,
provided: body.timestamp
})
}
// Store in cache for immediate use
const marketCache = getMarketDataCache()
@@ -115,7 +128,7 @@ export async function POST(request: NextRequest) {
pricePosition: Number(body.pricePosition) || 50,
maGap: Number(body.maGap) || undefined,
volume: Number(body.volume) || undefined,
timestamp: new Date(body.timestamp || Date.now())
timestamp
}
})

6
app/api/trading/reduce-position/route.ts
View File

@@ -189,6 +189,10 @@ export async function POST(request: NextRequest): Promise<NextResponse<ReducePos
const newTP1 = calculatePrice(trade.entryPrice, config.takeProfit1Percent, trade.direction)
const newTP2 = calculatePrice(trade.entryPrice, config.takeProfit2Percent, trade.direction)
const newSL = calculatePrice(trade.entryPrice, config.stopLossPercent, trade.direction)
const effectiveTp2SizePercent =
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0)
console.log(`🎯 New targets (same entry, reduced size):`)
console.log(` TP1: $${newTP1} (${config.takeProfit1Percent}%)`)
@@ -206,7 +210,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<ReducePos
tp2Price: newTP2,
stopLossPrice: newSL,
tp1SizePercent: config.takeProfit1SizePercent,
tp2SizePercent: config.takeProfit2SizePercent,
tp2SizePercent: effectiveTp2SizePercent,
useDualStops: config.useDualStops,
softStopPrice: config.useDualStops ? calculatePrice(trade.entryPrice, config.softStopPercent, trade.direction) : undefined,
softStopBuffer: config.useDualStops ? config.softStopBuffer : undefined,

6
app/api/trading/scale-position/route.ts
View File

@@ -156,6 +156,10 @@ export async function POST(request: NextRequest): Promise<NextResponse<ScalePosi
const newTP1 = calculatePrice(newAvgEntry, config.takeProfit1Percent, trade.direction)
const newTP2 = calculatePrice(newAvgEntry, config.takeProfit2Percent, trade.direction)
const newSL = calculatePrice(newAvgEntry, config.stopLossPercent, trade.direction)
const effectiveTp2SizePercent =
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0)
console.log(`🎯 New targets:`)
console.log(` TP1: $${newTP1} (${config.takeProfit1Percent}%)`)
@@ -173,7 +177,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<ScalePosi
tp2Price: newTP2,
stopLossPrice: newSL,
tp1SizePercent: config.takeProfit1SizePercent,
tp2SizePercent: config.takeProfit2SizePercent,
tp2SizePercent: effectiveTp2SizePercent,
useDualStops: config.useDualStops,
softStopPrice: config.useDualStops ? calculatePrice(newAvgEntry, config.softStopPercent, trade.direction) : undefined,
softStopBuffer: config.useDualStops ? config.softStopBuffer : undefined,

163
app/api/trading/sync-positions/route.ts
View File

@@ -31,7 +31,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
console.log(`📊 Found ${driftPositions.length} positions on Drift`)
// Get all currently tracked positions
const trackedTrades = Array.from(positionManager.getActiveTrades().values())
let trackedTrades = Array.from(positionManager.getActiveTrades().values())
console.log(`📋 Position Manager tracking ${trackedTrades.length} trades`)
const syncResults = {
@@ -71,6 +71,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
}
// Step 2: Add Drift positions that aren't being tracked
trackedTrades = Array.from(positionManager.getActiveTrades().values())
for (const driftPos of driftPositions) {
const isTracked = trackedTrades.some(t => t.symbol === driftPos.symbol)
@@ -99,48 +100,126 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
const tp2Price = calculatePrice(entryPrice, config.takeProfit2Percent, direction)
const emergencyStopPrice = calculatePrice(entryPrice, config.emergencyStopPercent, direction)
// Calculate position size in USD
const positionSizeUSD = driftPos.size * currentPrice
// Calculate position size in USD (Drift size is tokens)
const positionSizeUSD = Math.abs(driftPos.size) * currentPrice
// Create ActiveTrade object
const activeTrade = {
id: `sync-${Date.now()}-${driftPos.symbol}`,
positionId: `manual-${Date.now()}`, // Synthetic ID since we don't have the original
symbol: driftPos.symbol,
direction: direction,
entryPrice: entryPrice,
entryTime: Date.now() - (60 * 60 * 1000), // Assume 1 hour ago (we don't know actual time)
positionSize: positionSizeUSD,
leverage: config.leverage,
stopLossPrice: stopLossPrice,
tp1Price: tp1Price,
tp2Price: tp2Price,
emergencyStopPrice: emergencyStopPrice,
currentSize: positionSizeUSD,
originalPositionSize: positionSizeUSD, // Store original size for P&L
takeProfitPrice1: tp1Price,
takeProfitPrice2: tp2Price,
tp1Hit: false,
tp2Hit: false,
slMovedToBreakeven: false,
slMovedToProfit: false,
trailingStopActive: false,
realizedPnL: 0,
unrealizedPnL: driftPos.unrealizedPnL,
peakPnL: driftPos.unrealizedPnL,
peakPrice: currentPrice,
maxFavorableExcursion: 0,
maxAdverseExcursion: 0,
maxFavorablePrice: currentPrice,
maxAdversePrice: currentPrice,
originalAdx: undefined,
timesScaled: 0,
totalScaleAdded: 0,
atrAtEntry: undefined,
runnerTrailingPercent: undefined,
priceCheckCount: 0,
lastPrice: currentPrice,
lastUpdateTime: Date.now(),
// Try to find an existing open trade in the database for this symbol
const existingTrade = await prisma.trade.findFirst({
where: {
symbol: driftPos.symbol,
status: 'open',
},
orderBy: { entryTime: 'desc' },
})
const normalizeDirection = (dir: string): 'long' | 'short' =>
dir === 'long' ? 'long' : 'short'
const buildActiveTradeFromDb = (dbTrade: any): any => {
const pmState = (dbTrade.configSnapshot as any)?.positionManagerState
return {
id: dbTrade.id,
positionId: dbTrade.positionId,
symbol: dbTrade.symbol,
direction: normalizeDirection(dbTrade.direction),
entryPrice: dbTrade.entryPrice,
entryTime: dbTrade.entryTime.getTime(),
positionSize: dbTrade.positionSizeUSD,
leverage: dbTrade.leverage,
stopLossPrice: pmState?.stopLossPrice ?? dbTrade.stopLossPrice,
tp1Price: dbTrade.takeProfit1Price,
tp2Price: dbTrade.takeProfit2Price,
emergencyStopPrice: dbTrade.stopLossPrice * (dbTrade.direction === 'long' ? 0.98 : 1.02),
currentSize: pmState?.currentSize ?? dbTrade.positionSizeUSD,
originalPositionSize: dbTrade.positionSizeUSD,
takeProfitPrice1: dbTrade.takeProfit1Price,
takeProfitPrice2: dbTrade.takeProfit2Price,
tp1Hit: pmState?.tp1Hit ?? false,
tp2Hit: pmState?.tp2Hit ?? false,
slMovedToBreakeven: pmState?.slMovedToBreakeven ?? false,
slMovedToProfit: pmState?.slMovedToProfit ?? false,
trailingStopActive: pmState?.trailingStopActive ?? false,
realizedPnL: pmState?.realizedPnL ?? 0,
unrealizedPnL: pmState?.unrealizedPnL ?? 0,
peakPnL: pmState?.peakPnL ?? 0,
peakPrice: pmState?.peakPrice ?? dbTrade.entryPrice,
maxFavorableExcursion: pmState?.maxFavorableExcursion ?? 0,
maxAdverseExcursion: pmState?.maxAdverseExcursion ?? 0,
maxFavorablePrice: pmState?.maxFavorablePrice ?? dbTrade.entryPrice,
maxAdversePrice: pmState?.maxAdversePrice ?? dbTrade.entryPrice,
originalAdx: dbTrade.adxAtEntry,
timesScaled: pmState?.timesScaled ?? 0,
totalScaleAdded: pmState?.totalScaleAdded ?? 0,
atrAtEntry: dbTrade.atrAtEntry,
runnerTrailingPercent: pmState?.runnerTrailingPercent,
priceCheckCount: 0,
lastPrice: pmState?.lastPrice ?? dbTrade.entryPrice,
lastUpdateTime: Date.now(),
}
}
let activeTrade
if (existingTrade) {
console.log(`🔗 Found existing open trade in DB for ${driftPos.symbol}, attaching to Position Manager`)
activeTrade = buildActiveTradeFromDb(existingTrade)
} else {
console.warn(`⚠️ No open DB trade found for ${driftPos.symbol}. Creating synced placeholder to restore protection.`)
const now = new Date()
const syntheticPositionId = `sync-${now.getTime()}-${driftPos.marketIndex}`
const placeholderTrade = await prisma.trade.create({
data: {
positionId: syntheticPositionId,
symbol: driftPos.symbol,
direction,
entryPrice,
entryTime: now,
positionSizeUSD,
collateralUSD: positionSizeUSD / config.leverage,
leverage: config.leverage,
stopLossPrice,
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent,
tp2SizePercent:
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0),
status: 'open',
signalSource: 'drift_sync',
timeframe: 'sync',
configSnapshot: {
source: 'sync-positions',
syncedAt: now.toISOString(),
positionManagerState: {
currentSize: positionSizeUSD,
tp1Hit: false,
slMovedToBreakeven: false,
slMovedToProfit: false,
stopLossPrice,
realizedPnL: 0,
unrealizedPnL: driftPos.unrealizedPnL ?? 0,
peakPnL: driftPos.unrealizedPnL ?? 0,
lastPrice: currentPrice,
maxFavorableExcursion: 0,
maxAdverseExcursion: 0,
maxFavorablePrice: entryPrice,
maxAdversePrice: entryPrice,
lastUpdate: now.toISOString(),
},
},
entryOrderTx: syntheticPositionId,
},
})
const verifiedPlaceholder = await prisma.trade.findUnique({ where: { positionId: syntheticPositionId } })
if (!verifiedPlaceholder) {
throw new Error(`Placeholder trade not persisted for ${driftPos.symbol} (positionId=${syntheticPositionId})`)
}
activeTrade = buildActiveTradeFromDb(verifiedPlaceholder)
}
await positionManager.addTrade(activeTrade)

9
app/api/trading/test-db/route.ts
View File

@@ -128,6 +128,11 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
console.log(` TP1: $${tp1Price.toFixed(4)} (${config.takeProfit1Percent}%)`)
console.log(` TP2: $${tp2Price.toFixed(4)} (${config.takeProfit2Percent}%)`)
const effectiveTp2SizePercent =
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0)
// Place exit orders
let exitOrderSignatures: string[] = []
try {
@@ -139,7 +144,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
tp2Price,
stopLossPrice,
tp1SizePercent: config.takeProfit1SizePercent || 50,
tp2SizePercent: config.takeProfit2SizePercent || 100,
tp2SizePercent: effectiveTp2SizePercent,
direction,
useDualStops: config.useDualStops,
softStopPrice: config.useDualStops ? calculatePrice(entryPrice, config.softStopPercent, direction) : undefined,
@@ -210,7 +215,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent || 50,
tp2SizePercent: config.takeProfit2SizePercent || 100,
tp2SizePercent: effectiveTp2SizePercent,
configSnapshot: config,
entryOrderTx: openResult.transactionSignature!,
tp1OrderTx: exitOrderSignatures[0],

10
app/api/trading/test/route.ts
View File

@@ -267,8 +267,14 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
}
// Place on-chain TP/SL orders so they appear in Drift UI
let effectiveTp2SizePercent = 0
let exitOrderSignatures: string[] = []
try {
effectiveTp2SizePercent =
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0)
const exitRes = await placeExitOrders({
symbol: driftSymbol,
positionSizeUSD: actualPositionSizeUSD,
@@ -277,7 +283,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
tp2Price,
stopLossPrice,
tp1SizePercent: config.takeProfit1SizePercent ?? 50,
tp2SizePercent: config.takeProfit2SizePercent ?? 0, // 0 = activate trailing stop for runner
tp2SizePercent: effectiveTp2SizePercent, // 0 = activate trailing stop for runner
direction: direction,
// Dual stop parameters
useDualStops: config.useDualStops,
@@ -314,7 +320,7 @@ export async function POST(request: NextRequest): Promise<NextResponse<TestTrade
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent ?? 50,
tp2SizePercent: config.takeProfit2SizePercent ?? 0, // 0 = activate trailing stop for runner
tp2SizePercent: effectiveTp2SizePercent, // 0 = activate trailing stop for runner
configSnapshot: config,
entryOrderTx: openResult.transactionSignature!,
tp1OrderTx: exitOrderSignatures[0],