feat: Deploy HA auto-failover with database promotion

- Enhanced DNS failover monitor on secondary (72.62.39.24)
- Auto-promotes database: pg_ctl promote on failover
- Creates DEMOTED flag on primary via SSH (split-brain protection)
- Telegram notifications with database promotion status
- Startup safety script ready (integration pending)
- 90-second automatic recovery vs 10-30 min manual
- Zero-cost 95% enterprise HA benefit

Status: DEPLOYED and MONITORING (14:52 CET)
Next: Controlled failover test during maintenance
This commit is contained in:
mindesbunister
2025-12-12 15:54:03 +01:00
parent 7ff5c5b3a4
commit d637aac2d7
25 changed files with 1071 additions and 170 deletions

View File

@@ -54,6 +54,8 @@ export interface PlaceExitOrdersResult {
success: boolean
signatures?: string[]
error?: string
expectedOrders?: number
placedOrders?: number
}
export interface PlaceExitOrdersOptions {
@@ -271,6 +273,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
}
const signatures: string[] = []
let expectedOrders = 0
// Helper to compute base asset amount from USD notional and price
// CRITICAL FIX (Dec 10, 2025): Must use SPECIFIC PRICE for each order (TP1 price, TP2 price, SL price)
@@ -285,9 +288,16 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
// CRITICAL FIX: TP2 must be percentage of REMAINING size after TP1, not original size
const tp1USD = (options.positionSizeUSD * options.tp1SizePercent) / 100
const remainingAfterTP1 = options.positionSizeUSD - tp1USD
const requestedTp2Percent = options.tp2SizePercent ?? 100
const normalizedTp2Percent = requestedTp2Percent > 0 ? requestedTp2Percent : 100
const requestedTp2Percent = options.tp2SizePercent
// Allow explicit 0% to mean "no TP2 order" without forcing it back to 100%
const normalizedTp2Percent = requestedTp2Percent === undefined
? 100
: Math.max(0, requestedTp2Percent)
const tp2USD = (remainingAfterTP1 * normalizedTp2Percent) / 100
if (normalizedTp2Percent === 0) {
logger.log(' TP2 on-chain order skipped (trigger-only; software handles trailing)')
}
logger.log(`📊 Exit order sizes:`)
logger.log(` TP1: ${options.tp1SizePercent}% of $${options.positionSizeUSD.toFixed(2)} = $${tp1USD.toFixed(2)}`)
@@ -302,6 +312,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
if (tp1USD > 0) {
const baseAmount = usdToBase(tp1USD, options.tp1Price) // Use TP1 price
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
expectedOrders += 1
const orderParams: any = {
orderType: OrderType.LIMIT,
marketIndex: marketConfig.driftMarketIndex,
@@ -326,6 +337,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
if (tp2USD > 0) {
const baseAmount = usdToBase(tp2USD, options.tp2Price) // Use TP2 price
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
expectedOrders += 1
const orderParams: any = {
orderType: OrderType.LIMIT,
marketIndex: marketConfig.driftMarketIndex,
@@ -355,17 +367,22 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
const slUSD = options.positionSizeUSD
const slBaseAmount = usdToBase(slUSD, options.stopLossPrice) // Use SL price
// Calculate expected number of orders for validation (Bug #76 fix)
const useDualStops = options.useDualStops ?? false
const expectedOrderCount = 2 + (useDualStops ? 2 : 1) // TP1 + TP2 + (soft+hard SL OR single SL)
logger.log(`📊 Expected exit orders: TP1/TP2 that meet min size + ${useDualStops ? 'soft+hard SL' : 'single SL'}`)
logger.log(`📊 Expected ${expectedOrderCount} exit orders total (TP1 + TP2 + ${useDualStops ? 'dual stops' : 'single stop'})`)
const minOrderLamports = Math.floor(marketConfig.minOrderSize * 1e9)
if (slBaseAmount < minOrderLamports) {
const errorMsg = `Stop loss size below market minimum (base ${slBaseAmount} < min ${minOrderLamports})`
console.error(`${errorMsg}`)
return { success: false, error: errorMsg, signatures, expectedOrders, placedOrders: signatures.length }
}
if (slBaseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
if (slBaseAmount >= minOrderLamports) {
if (useDualStops && options.softStopPrice && options.hardStopPrice) {
// ============== DUAL STOP SYSTEM ==============
logger.log('🛡️🛡️ Placing DUAL STOP SYSTEM...')
expectedOrders += 2
try {
// 1. Soft Stop (TRIGGER_LIMIT) - Avoids wicks
@@ -438,6 +455,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
// ============== SINGLE STOP SYSTEM ==============
const useStopLimit = options.useStopLimit ?? false
const stopLimitBuffer = options.stopLimitBuffer ?? 0.5
expectedOrders += 1
try {
if (useStopLimit) {
@@ -500,26 +518,26 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
throw new Error(`Stop loss placement failed: ${slError instanceof Error ? slError.message : 'Unknown error'}`)
}
}
} else {
logger.log('⚠️ SL size below market min, skipping on-chain SL')
}
// CRITICAL VALIDATION (Bug #76 fix): Verify all expected orders were placed
if (signatures.length < expectedOrderCount) {
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrderCount}, got ${signatures.length}. Position is UNPROTECTED!`
const placedOrders = signatures.length
if (placedOrders < expectedOrders) {
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrders}, got ${placedOrders}. Position is UNPROTECTED!`
console.error(`${errorMsg}`)
console.error(` Expected: TP1 + TP2 + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
console.error(` Got ${signatures.length} signatures:`, signatures)
console.error(` Expected: TP1/TP2 that met min + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
console.error(` Got ${placedOrders} signatures:`, signatures)
return {
success: false,
error: errorMsg,
signatures // Return partial signatures for debugging
signatures, // Return partial signatures for debugging
expectedOrders,
placedOrders,
}
}
logger.log(`✅ All ${expectedOrderCount} exit orders placed successfully`)
return { success: true, signatures }
logger.log(`✅ All ${expectedOrders} exit orders placed successfully`)
return { success: true, signatures, expectedOrders, placedOrders }
} catch (error) {
console.error('❌ Failed to place exit orders:', error)
return { success: false, error: error instanceof Error ? error.message : 'Unknown error' }
@@ -533,20 +551,20 @@ export async function closePosition(
params: ClosePositionParams
): Promise<ClosePositionResult> {
try {
logger.log('📊 Closing position:', params)
const driftService = getDriftService()
const marketConfig = getMarketConfig(params.symbol)
const driftService = await getDriftService()
const driftClient = driftService.getClient()
// Get current position
const position = await driftService.getPosition(marketConfig.driftMarketIndex)
if (!position || position.side === 'none') {
throw new Error(`No active position for ${params.symbol}`)
console.warn(`⚠️ No open position found for ${params.symbol}, skipping close request`)
return {
success: false,
error: 'No open position to close',
}
}
console.log(`🔍 CLOSE POSITION DEBUG:`)
logger.log('📊 Closing position:', params)
console.log(` params.percentToClose: ${params.percentToClose}`)
console.log(` position.size: ${position.size}`)
console.log(` marketConfig.minOrderSize: ${marketConfig.minOrderSize}`)
@@ -627,6 +645,7 @@ export async function closePosition(
// BUT: Use timeout to prevent API hangs during network congestion
logger.log('⏳ Confirming transaction on-chain (30s timeout)...')
const connection = driftService.getTradeConnection() // Use Alchemy for trade operations
let confirmationTimedOut = false
try {
const confirmationPromise = connection.confirmTransaction(txSig, 'confirmed')
@@ -644,10 +663,11 @@ export async function closePosition(
logger.log('✅ Transaction confirmed on-chain')
} catch (timeoutError: any) {
if (timeoutError.message === 'Transaction confirmation timeout') {
confirmationTimedOut = true
console.warn('⚠️ Transaction confirmation timed out after 30s')
console.warn(' Order may still execute - check Drift UI')
console.warn(` Transaction signature: ${txSig}`)
// Continue anyway - order was submitted and will likely execute
// Continue but flag for Position Manager verification so we do not drop tracking
} else {
throw timeoutError
}
@@ -726,6 +746,7 @@ export async function closePosition(
closePrice: oraclePrice,
closedSize: sizeToClose,
realizedPnL,
needsVerification: confirmationTimedOut, // Keep monitoring if confirmation never arrived
}
} catch (error) {