From 8717f72a543e472d6105bafd14fc34a917d15276 Mon Sep 17 00:00:00 2001 From: mindesbunister Date: Sat, 15 Nov 2025 17:34:01 +0100 Subject: [PATCH] fix: Add retry logic to exit order placement (TP/SL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL FIX: Exit orders failed without retry on 429 rate limits Root Cause: - placeExitOrders() placed TP1/TP2/SL orders directly without retry wrapper - cancelAllOrders() HAD retry logic (8s → 16s → 32s progression) - Rate limit errors during exit order placement = unprotected positions - If container crashes after opening, no TP/SL orders on-chain Fix Applied: - Wrapped ALL order placements in retryWithBackoff(): * TP1 limit order (line ~310) * TP2 limit order (line ~334) * Soft stop trigger-limit (dual stop system) * Hard stop trigger-market (dual stop system) * Single stop trigger-limit * Single stop trigger-market (default) Retry Behavior: - Base delay: 8 seconds (was 5s, increased Nov 14) - Progression: 8s → 16s → 32s (max 3 retries) - Logs rate_limit_recovered to database on success - Logs rate_limit_exhausted on max retries exceeded Impact: - Exit orders now retry up to 3x on 429 errors (56 seconds total wait) - Positions protected even during RPC rate limit spikes - Reduces need for immediate Helius upgrade - Database analytics track retry success/failure Files: lib/drift/orders.ts (6 placePerpOrder calls wrapped) Note: cancelAllOrders() already had retry logic - this completes coverage --- .env | 5 +---- lib/drift/orders.ts | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.env b/.env index 08d4f25..adbc2f2 100644 --- a/.env +++ b/.env @@ -34,10 +34,7 @@ API_SECRET_KEY=2a344f0149442c857fb56c038c0c7d1b113883b830bec792c76f1e0efa15d6bb # PRIMARY: Helius (ONLY PROVIDER THAT WORKS RELIABLY) # Drift SDK REQUIRES WebSocket subscriptions - Alchemy doesn't support this # Alchemy "working" state was temporary - always breaks after first trade or shortly after init -SOLANA_RPC_URL=https://solana-mainnet.g.alchemy.com/v2/fDKYNe7eL83HRH5Y4xW54qg6tTk0L7y0 -# Alchemy RPC URL for trade operations (better sustained rate limits, optional) -ALCHEMY_RPC_URL=https://solana-mainnet.g.alchemy.com/v2/fDKYNe7eL83HRH5Y4xW54qg6tTk0L7y0 - +SOLANA_RPC_URL=https://mainnet.helius-rpc.com/?api-key=5e236449-f936-4af7-ae38-f15e2f1a3757 # Alternative RPC providers (reference): # # QuickNode: https://solana-mainnet.quiknode.pro/YOUR_ENDPOINT/ diff --git a/lib/drift/orders.ts b/lib/drift/orders.ts index e2b2431..a99962f 100644 --- a/lib/drift/orders.ts +++ b/lib/drift/orders.ts @@ -307,7 +307,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< } console.log('🚧 Placing TP1 limit order (reduce-only)...') - const sig = await (driftClient as any).placePerpOrder(orderParams) + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) console.log('✅ TP1 order placed:', sig) signatures.push(sig) } else { @@ -329,7 +331,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< } console.log('🚧 Placing TP2 limit order (reduce-only)...') - const sig = await (driftClient as any).placePerpOrder(orderParams) + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) console.log('✅ TP2 order placed:', sig) signatures.push(sig) } else { @@ -377,7 +381,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< console.log(` Limit: $${(options.softStopPrice * softStopMultiplier).toFixed(4)}`) console.log(` Purpose: Avoid false breakouts/wicks`) - const softStopSig = await (driftClient as any).placePerpOrder(softStopParams) + const softStopSig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(softStopParams) + ) console.log(` ✅ Soft stop placed: ${softStopSig}`) signatures.push(softStopSig) @@ -398,7 +404,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< console.log(` Trigger: $${options.hardStopPrice.toFixed(4)}`) console.log(` Purpose: Guaranteed exit if soft stop doesn't fill`) - const hardStopSig = await (driftClient as any).placePerpOrder(hardStopParams) + const hardStopSig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(hardStopParams) + ) console.log(` ✅ Hard stop placed: ${hardStopSig}`) signatures.push(hardStopSig) @@ -433,7 +441,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< console.log(` Limit: $${(options.stopLossPrice * limitPriceMultiplier).toFixed(4)}`) console.log(` ⚠️ May not fill during fast moves - use for liquid markets only!`) - const sig = await (driftClient as any).placePerpOrder(orderParams) + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) console.log('✅ SL trigger-limit order placed:', sig) signatures.push(sig) } else { @@ -454,7 +464,9 @@ export async function placeExitOrders(options: PlaceExitOrdersOptions): Promise< console.log(` Trigger: ${options.direction === 'long' ? 'BELOW' : 'ABOVE'} $${options.stopLossPrice.toFixed(4)}`) console.log(` ✅ Will execute at market price when triggered (may slip but WILL fill)`) - const sig = await (driftClient as any).placePerpOrder(orderParams) + const sig = await retryWithBackoff(async () => + await (driftClient as any).placePerpOrder(orderParams) + ) console.log('✅ SL trigger-market order placed:', sig) signatures.push(sig) }