feat: Deploy HA auto-failover with database promotion
- Enhanced DNS failover monitor on secondary (72.62.39.24) - Auto-promotes database: pg_ctl promote on failover - Creates DEMOTED flag on primary via SSH (split-brain protection) - Telegram notifications with database promotion status - Startup safety script ready (integration pending) - 90-second automatic recovery vs 10-30 min manual - Zero-cost 95% enterprise HA benefit Status: DEPLOYED and MONITORING (14:52 CET) Next: Controlled failover test during maintenance
This commit is contained in:
@@ -54,6 +54,8 @@ export interface PlaceExitOrdersResult {
|
||||
success: boolean
|
||||
signatures?: string[]
|
||||
error?: string
|
||||
expectedOrders?: number
|
||||
placedOrders?: number
|
||||
}
|
||||
|
||||
export interface PlaceExitOrdersOptions {
|
||||
@@ -271,6 +273,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
}
|
||||
|
||||
const signatures: string[] = []
|
||||
let expectedOrders = 0
|
||||
|
||||
// Helper to compute base asset amount from USD notional and price
|
||||
// CRITICAL FIX (Dec 10, 2025): Must use SPECIFIC PRICE for each order (TP1 price, TP2 price, SL price)
|
||||
@@ -285,9 +288,16 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
// CRITICAL FIX: TP2 must be percentage of REMAINING size after TP1, not original size
|
||||
const tp1USD = (options.positionSizeUSD * options.tp1SizePercent) / 100
|
||||
const remainingAfterTP1 = options.positionSizeUSD - tp1USD
|
||||
const requestedTp2Percent = options.tp2SizePercent ?? 100
|
||||
const normalizedTp2Percent = requestedTp2Percent > 0 ? requestedTp2Percent : 100
|
||||
const requestedTp2Percent = options.tp2SizePercent
|
||||
// Allow explicit 0% to mean "no TP2 order" without forcing it back to 100%
|
||||
const normalizedTp2Percent = requestedTp2Percent === undefined
|
||||
? 100
|
||||
: Math.max(0, requestedTp2Percent)
|
||||
const tp2USD = (remainingAfterTP1 * normalizedTp2Percent) / 100
|
||||
|
||||
if (normalizedTp2Percent === 0) {
|
||||
logger.log('ℹ️ TP2 on-chain order skipped (trigger-only; software handles trailing)')
|
||||
}
|
||||
|
||||
logger.log(`📊 Exit order sizes:`)
|
||||
logger.log(` TP1: ${options.tp1SizePercent}% of $${options.positionSizeUSD.toFixed(2)} = $${tp1USD.toFixed(2)}`)
|
||||
@@ -302,6 +312,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
if (tp1USD > 0) {
|
||||
const baseAmount = usdToBase(tp1USD, options.tp1Price) // Use TP1 price
|
||||
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
|
||||
expectedOrders += 1
|
||||
const orderParams: any = {
|
||||
orderType: OrderType.LIMIT,
|
||||
marketIndex: marketConfig.driftMarketIndex,
|
||||
@@ -326,6 +337,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
if (tp2USD > 0) {
|
||||
const baseAmount = usdToBase(tp2USD, options.tp2Price) // Use TP2 price
|
||||
if (baseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
|
||||
expectedOrders += 1
|
||||
const orderParams: any = {
|
||||
orderType: OrderType.LIMIT,
|
||||
marketIndex: marketConfig.driftMarketIndex,
|
||||
@@ -355,17 +367,22 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
const slUSD = options.positionSizeUSD
|
||||
const slBaseAmount = usdToBase(slUSD, options.stopLossPrice) // Use SL price
|
||||
|
||||
// Calculate expected number of orders for validation (Bug #76 fix)
|
||||
const useDualStops = options.useDualStops ?? false
|
||||
const expectedOrderCount = 2 + (useDualStops ? 2 : 1) // TP1 + TP2 + (soft+hard SL OR single SL)
|
||||
logger.log(`📊 Expected exit orders: TP1/TP2 that meet min size + ${useDualStops ? 'soft+hard SL' : 'single SL'}`)
|
||||
|
||||
logger.log(`📊 Expected ${expectedOrderCount} exit orders total (TP1 + TP2 + ${useDualStops ? 'dual stops' : 'single stop'})`)
|
||||
const minOrderLamports = Math.floor(marketConfig.minOrderSize * 1e9)
|
||||
if (slBaseAmount < minOrderLamports) {
|
||||
const errorMsg = `Stop loss size below market minimum (base ${slBaseAmount} < min ${minOrderLamports})`
|
||||
console.error(`❌ ${errorMsg}`)
|
||||
return { success: false, error: errorMsg, signatures, expectedOrders, placedOrders: signatures.length }
|
||||
}
|
||||
|
||||
if (slBaseAmount >= Math.floor(marketConfig.minOrderSize * 1e9)) {
|
||||
if (slBaseAmount >= minOrderLamports) {
|
||||
|
||||
if (useDualStops && options.softStopPrice && options.hardStopPrice) {
|
||||
// ============== DUAL STOP SYSTEM ==============
|
||||
logger.log('🛡️🛡️ Placing DUAL STOP SYSTEM...')
|
||||
expectedOrders += 2
|
||||
|
||||
try {
|
||||
// 1. Soft Stop (TRIGGER_LIMIT) - Avoids wicks
|
||||
@@ -438,6 +455,7 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
// ============== SINGLE STOP SYSTEM ==============
|
||||
const useStopLimit = options.useStopLimit ?? false
|
||||
const stopLimitBuffer = options.stopLimitBuffer ?? 0.5
|
||||
expectedOrders += 1
|
||||
|
||||
try {
|
||||
if (useStopLimit) {
|
||||
@@ -500,26 +518,26 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise<
|
||||
throw new Error(`Stop loss placement failed: ${slError instanceof Error ? slError.message : 'Unknown error'}`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.log('⚠️ SL size below market min, skipping on-chain SL')
|
||||
}
|
||||
|
||||
// CRITICAL VALIDATION (Bug #76 fix): Verify all expected orders were placed
|
||||
if (signatures.length < expectedOrderCount) {
|
||||
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrderCount}, got ${signatures.length}. Position is UNPROTECTED!`
|
||||
const placedOrders = signatures.length
|
||||
if (placedOrders < expectedOrders) {
|
||||
const errorMsg = `MISSING EXIT ORDERS: Expected ${expectedOrders}, got ${placedOrders}. Position is UNPROTECTED!`
|
||||
console.error(`❌ ${errorMsg}`)
|
||||
console.error(` Expected: TP1 + TP2 + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
|
||||
console.error(` Got ${signatures.length} signatures:`, signatures)
|
||||
console.error(` Expected: TP1/TP2 that met min + ${useDualStops ? 'Soft SL + Hard SL' : 'SL'}`)
|
||||
console.error(` Got ${placedOrders} signatures:`, signatures)
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMsg,
|
||||
signatures // Return partial signatures for debugging
|
||||
signatures, // Return partial signatures for debugging
|
||||
expectedOrders,
|
||||
placedOrders,
|
||||
}
|
||||
}
|
||||
|
||||
logger.log(`✅ All ${expectedOrderCount} exit orders placed successfully`)
|
||||
return { success: true, signatures }
|
||||
logger.log(`✅ All ${expectedOrders} exit orders placed successfully`)
|
||||
return { success: true, signatures, expectedOrders, placedOrders }
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to place exit orders:', error)
|
||||
return { success: false, error: error instanceof Error ? error.message : 'Unknown error' }
|
||||
@@ -533,20 +551,20 @@ export async function closePosition(
|
||||
params: ClosePositionParams
|
||||
): Promise<ClosePositionResult> {
|
||||
try {
|
||||
logger.log('📊 Closing position:', params)
|
||||
|
||||
const driftService = getDriftService()
|
||||
const marketConfig = getMarketConfig(params.symbol)
|
||||
const driftService = await getDriftService()
|
||||
const driftClient = driftService.getClient()
|
||||
|
||||
// Get current position
|
||||
const position = await driftService.getPosition(marketConfig.driftMarketIndex)
|
||||
|
||||
|
||||
if (!position || position.side === 'none') {
|
||||
throw new Error(`No active position for ${params.symbol}`)
|
||||
console.warn(`⚠️ No open position found for ${params.symbol}, skipping close request`)
|
||||
return {
|
||||
success: false,
|
||||
error: 'No open position to close',
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`🔍 CLOSE POSITION DEBUG:`)
|
||||
|
||||
logger.log('📊 Closing position:', params)
|
||||
console.log(` params.percentToClose: ${params.percentToClose}`)
|
||||
console.log(` position.size: ${position.size}`)
|
||||
console.log(` marketConfig.minOrderSize: ${marketConfig.minOrderSize}`)
|
||||
@@ -627,6 +645,7 @@ export async function closePosition(
|
||||
// BUT: Use timeout to prevent API hangs during network congestion
|
||||
logger.log('⏳ Confirming transaction on-chain (30s timeout)...')
|
||||
const connection = driftService.getTradeConnection() // Use Alchemy for trade operations
|
||||
let confirmationTimedOut = false
|
||||
|
||||
try {
|
||||
const confirmationPromise = connection.confirmTransaction(txSig, 'confirmed')
|
||||
@@ -644,10 +663,11 @@ export async function closePosition(
|
||||
logger.log('✅ Transaction confirmed on-chain')
|
||||
} catch (timeoutError: any) {
|
||||
if (timeoutError.message === 'Transaction confirmation timeout') {
|
||||
confirmationTimedOut = true
|
||||
console.warn('⚠️ Transaction confirmation timed out after 30s')
|
||||
console.warn(' Order may still execute - check Drift UI')
|
||||
console.warn(` Transaction signature: ${txSig}`)
|
||||
// Continue anyway - order was submitted and will likely execute
|
||||
// Continue but flag for Position Manager verification so we do not drop tracking
|
||||
} else {
|
||||
throw timeoutError
|
||||
}
|
||||
@@ -726,6 +746,7 @@ export async function closePosition(
|
||||
closePrice: oraclePrice,
|
||||
closedSize: sizeToClose,
|
||||
realizedPnL,
|
||||
needsVerification: confirmationTimedOut, // Keep monitoring if confirmation never arrived
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user