root/trading_bot_v4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Deploy HA auto-failover with database promotion

Browse Source 
- Enhanced DNS failover monitor on secondary (72.62.39.24)
- Auto-promotes database: pg_ctl promote on failover
- Creates DEMOTED flag on primary via SSH (split-brain protection)
- Telegram notifications with database promotion status
- Startup safety script ready (integration pending)
- 90-second automatic recovery vs 10-30 min manual
- Zero-cost 95% enterprise HA benefit

Status: DEPLOYED and MONITORING (14:52 CET)
Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
2025-12-12 15:54:03 +01:00
parent 7ff5c5b3a4
commit d637aac2d7
25 changed files with 1071 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
app/api/trading/sync-positions/route.ts
View File

@@ -31,7 +31,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
console.log(`📊 Found ${driftPositions.length} positions on Drift`)
// Get all currently tracked positions
const trackedTrades = Array.from(positionManager.getActiveTrades().values())
let trackedTrades = Array.from(positionManager.getActiveTrades().values())
console.log(`📋 Position Manager tracking ${trackedTrades.length} trades`)
const syncResults = {
@@ -71,6 +71,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
}
// Step 2: Add Drift positions that aren't being tracked
trackedTrades = Array.from(positionManager.getActiveTrades().values())
for (const driftPos of driftPositions) {
const isTracked = trackedTrades.some(t => t.symbol === driftPos.symbol)
@@ -99,48 +100,126 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
const tp2Price = calculatePrice(entryPrice, config.takeProfit2Percent, direction)
const emergencyStopPrice = calculatePrice(entryPrice, config.emergencyStopPercent, direction)
// Calculate position size in USD
const positionSizeUSD = driftPos.size * currentPrice
// Calculate position size in USD (Drift size is tokens)
const positionSizeUSD = Math.abs(driftPos.size) * currentPrice
// Create ActiveTrade object
const activeTrade = {
id: `sync-${Date.now()}-${driftPos.symbol}`,
positionId: `manual-${Date.now()}`, // Synthetic ID since we don't have the original
symbol: driftPos.symbol,
direction: direction,
entryPrice: entryPrice,
entryTime: Date.now() - (60 * 60 * 1000), // Assume 1 hour ago (we don't know actual time)
positionSize: positionSizeUSD,
leverage: config.leverage,
stopLossPrice: stopLossPrice,
tp1Price: tp1Price,
tp2Price: tp2Price,
emergencyStopPrice: emergencyStopPrice,
currentSize: positionSizeUSD,
originalPositionSize: positionSizeUSD, // Store original size for P&L
takeProfitPrice1: tp1Price,
takeProfitPrice2: tp2Price,
tp1Hit: false,
tp2Hit: false,
slMovedToBreakeven: false,
slMovedToProfit: false,
trailingStopActive: false,
realizedPnL: 0,
unrealizedPnL: driftPos.unrealizedPnL,
peakPnL: driftPos.unrealizedPnL,
peakPrice: currentPrice,
maxFavorableExcursion: 0,
maxAdverseExcursion: 0,
maxFavorablePrice: currentPrice,
maxAdversePrice: currentPrice,
originalAdx: undefined,
timesScaled: 0,
totalScaleAdded: 0,
atrAtEntry: undefined,
runnerTrailingPercent: undefined,
priceCheckCount: 0,
lastPrice: currentPrice,
lastUpdateTime: Date.now(),
// Try to find an existing open trade in the database for this symbol
const existingTrade = await prisma.trade.findFirst({
where: {
symbol: driftPos.symbol,
status: 'open',
},
orderBy: { entryTime: 'desc' },
})
const normalizeDirection = (dir: string): 'long' | 'short' =>
dir === 'long' ? 'long' : 'short'
const buildActiveTradeFromDb = (dbTrade: any): any => {
const pmState = (dbTrade.configSnapshot as any)?.positionManagerState
return {
id: dbTrade.id,
positionId: dbTrade.positionId,
symbol: dbTrade.symbol,
direction: normalizeDirection(dbTrade.direction),
entryPrice: dbTrade.entryPrice,
entryTime: dbTrade.entryTime.getTime(),
positionSize: dbTrade.positionSizeUSD,
leverage: dbTrade.leverage,
stopLossPrice: pmState?.stopLossPrice ?? dbTrade.stopLossPrice,
tp1Price: dbTrade.takeProfit1Price,
tp2Price: dbTrade.takeProfit2Price,
emergencyStopPrice: dbTrade.stopLossPrice * (dbTrade.direction === 'long' ? 0.98 : 1.02),
currentSize: pmState?.currentSize ?? dbTrade.positionSizeUSD,
originalPositionSize: dbTrade.positionSizeUSD,
takeProfitPrice1: dbTrade.takeProfit1Price,
takeProfitPrice2: dbTrade.takeProfit2Price,
tp1Hit: pmState?.tp1Hit ?? false,
tp2Hit: pmState?.tp2Hit ?? false,
slMovedToBreakeven: pmState?.slMovedToBreakeven ?? false,
slMovedToProfit: pmState?.slMovedToProfit ?? false,
trailingStopActive: pmState?.trailingStopActive ?? false,
realizedPnL: pmState?.realizedPnL ?? 0,
unrealizedPnL: pmState?.unrealizedPnL ?? 0,
peakPnL: pmState?.peakPnL ?? 0,
peakPrice: pmState?.peakPrice ?? dbTrade.entryPrice,
maxFavorableExcursion: pmState?.maxFavorableExcursion ?? 0,
maxAdverseExcursion: pmState?.maxAdverseExcursion ?? 0,
maxFavorablePrice: pmState?.maxFavorablePrice ?? dbTrade.entryPrice,
maxAdversePrice: pmState?.maxAdversePrice ?? dbTrade.entryPrice,
originalAdx: dbTrade.adxAtEntry,
timesScaled: pmState?.timesScaled ?? 0,
totalScaleAdded: pmState?.totalScaleAdded ?? 0,
atrAtEntry: dbTrade.atrAtEntry,
runnerTrailingPercent: pmState?.runnerTrailingPercent,
priceCheckCount: 0,
lastPrice: pmState?.lastPrice ?? dbTrade.entryPrice,
lastUpdateTime: Date.now(),
}
}
let activeTrade
if (existingTrade) {
console.log(`🔗 Found existing open trade in DB for ${driftPos.symbol}, attaching to Position Manager`)
activeTrade = buildActiveTradeFromDb(existingTrade)
} else {
console.warn(`⚠️ No open DB trade found for ${driftPos.symbol}. Creating synced placeholder to restore protection.`)
const now = new Date()
const syntheticPositionId = `sync-${now.getTime()}-${driftPos.marketIndex}`
const placeholderTrade = await prisma.trade.create({
data: {
positionId: syntheticPositionId,
symbol: driftPos.symbol,
direction,
entryPrice,
entryTime: now,
positionSizeUSD,
collateralUSD: positionSizeUSD / config.leverage,
leverage: config.leverage,
stopLossPrice,
takeProfit1Price: tp1Price,
takeProfit2Price: tp2Price,
tp1SizePercent: config.takeProfit1SizePercent,
tp2SizePercent:
config.useTp2AsTriggerOnly && (config.takeProfit2SizePercent ?? 0) <= 0
? 0
: (config.takeProfit2SizePercent ?? 0),
status: 'open',
signalSource: 'drift_sync',
timeframe: 'sync',
configSnapshot: {
source: 'sync-positions',
syncedAt: now.toISOString(),
positionManagerState: {
currentSize: positionSizeUSD,
tp1Hit: false,
slMovedToBreakeven: false,
slMovedToProfit: false,
stopLossPrice,
realizedPnL: 0,
unrealizedPnL: driftPos.unrealizedPnL ?? 0,
peakPnL: driftPos.unrealizedPnL ?? 0,
lastPrice: currentPrice,
maxFavorableExcursion: 0,
maxAdverseExcursion: 0,
maxFavorablePrice: entryPrice,
maxAdversePrice: entryPrice,
lastUpdate: now.toISOString(),
},
},
entryOrderTx: syntheticPositionId,
},
})
const verifiedPlaceholder = await prisma.trade.findUnique({ where: { positionId: syntheticPositionId } })
if (!verifiedPlaceholder) {
throw new Error(`Placeholder trade not persisted for ${driftPos.symbol} (positionId=${syntheticPositionId})`)
}
activeTrade = buildActiveTradeFromDb(verifiedPlaceholder)
}
await positionManager.addTrade(activeTrade)