feat: Hybrid RPC fallback system (Alchemy → Helius)

- Automatic fallback after 2 consecutive rate limits - Primary: Alchemy (300M CU/month, stable for normal ops) - Fallback: Helius (10 req/sec, backup for startup bursts) - Reduced startup validation: 6h window, 5 trades (was 24h, 20 trades) - Multi-position safety check (prevents order cancellation conflicts) - Rate limit-aware retry logic with exponential backoff Implementation: - lib/drift/client.ts: Added fallbackConnection, switchToFallbackRpc() - .env: SOLANA_FALLBACK_RPC_URL configuration - lib/startup/init-position-manager.ts: Reduced validation scope - lib/trading/position-manager.ts: Multi-position order protection Tested: System switched to fallback on startup, Position Manager active Result: 1 active trade being monitored after automatic RPC switch
2025-11-14 15:28:07 +01:00
parent d5183514bc
commit 7ff78ee0bd
5 changed files with 193 additions and 50 deletions
--- a/.env
+++ b/.env
@@ -31,16 +31,20 @@ API_SECRET_KEY=2a344f0149442c857fb56c038c0c7d1b113883b830bec792c76f1e0efa15d6bb

 # Solana RPC URL (Required for blockchain access)
 # 
-# RECOMMENDED: Helius (best performance, free tier available)
-# Get free API key at: https://helius.dev
+# CRITICAL: Primary RPC for all trading operations
+# Current: Alchemy (300M compute units/month free tier)
 SOLANA_RPC_URL=https://solana-mainnet.g.alchemy.com/v2/5A0iA5UYpsmP9gkuezYeg

-# Alternative RPC providers (if not using Helius):
-# 
-# QuickNode: https://solana-mainnet.quiknode.pro/YOUR_ENDPOINT/
-# Alchemy: https://solana-mainnet.g.alchemy.com/v2/YOUR_ALCHEMY_KEY
-# Ankr: https://rpc.ankr.com/solana
-# Public (not recommended): https://api.mainnet-beta.solana.com
+# Fallback RPC URL (Optional but HIGHLY recommended)
+# Automatically switches to fallback after 2 consecutive rate limits
+# Use a different provider than primary for best redundancy
+SOLANA_FALLBACK_RPC_URL=https://mainnet.helius-rpc.com/v1/?api-key=dcca4bf0-0b91-4f6a-8d12-1c5a4c1c6e5b
+
+# RPC Provider Comparison (as of Nov 2025):
+# ✅ Alchemy: 300M CU/month, excellent for primary (CURRENT PRIMARY)
+# ⚠️  Helius: 10 req/sec sustained (free tier), good for fallback only
+# QuickNode: Paid plans, very reliable
+# Ankr/Public: Unreliable, not recommended

 # ================================
 # REQUIRED - PYTH NETWORK (Price Feeds)
--- a/lib/drift/client.ts
+++ b/lib/drift/client.ts
@@ -17,20 +17,32 @@ interface ManualWallet {

 export interface DriftConfig {
  rpcUrl: string
+  fallbackRpcUrl?: string  // Optional fallback RPC for rate limit handling
  walletPrivateKey: string
  env: 'mainnet-beta' | 'devnet'
 }

 export class DriftService {
  private connection: Connection
+  private fallbackConnection: Connection | null = null
+  private currentRpcUrl: string
  private wallet: ManualWallet
  private keypair: Keypair
  private driftClient: DriftClient | null = null
  private user: User | null = null
  private isInitialized: boolean = false
+  private consecutiveRateLimits: number = 0
+  private usingFallback: boolean = false

  constructor(private config: DriftConfig) {
    this.connection = new Connection(config.rpcUrl, 'confirmed')
+    this.currentRpcUrl = config.rpcUrl
+    
+    // Initialize fallback connection if provided
+    if (config.fallbackRpcUrl) {
+      this.fallbackConnection = new Connection(config.fallbackRpcUrl, 'confirmed')
+      console.log('🔄 Fallback RPC configured:', this.maskRpcUrl(config.fallbackRpcUrl))
+    }
    
    // Create wallet from private key
    // Support both formats:
@@ -76,15 +88,53 @@ export class DriftService {
  }

  /**
-   * Retry helper for handling transient network failures (DNS, timeouts)
+   * Mask RPC URL for logging (hide API key)
+   */
+  private maskRpcUrl(url: string): string {
+    return url.replace(/\/v2\/[^/]+/, '/v2/***').replace(/api-key=[^&]+/, 'api-key=***')
+  }
+
+  /**
+   * Switch to fallback RPC if available
+   */
+  private switchToFallbackRpc(): boolean {
+    if (!this.fallbackConnection || this.usingFallback) {
+      return false // No fallback available or already using it
+    }
+
+    console.log('🔄 Switching from primary to fallback RPC')
+    this.connection = this.fallbackConnection
+    this.usingFallback = true
+    this.consecutiveRateLimits = 0
+    return true
+  }
+
+  /**
+   * Switch back to primary RPC
+   */
+  private switchToPrimaryRpc(): void {
+    if (!this.usingFallback) {
+      return // Already using primary
+    }
+
+    console.log('🔄 Switching back to primary RPC')
+    this.connection = new Connection(this.config.rpcUrl, 'confirmed')
+    this.usingFallback = false
+    this.consecutiveRateLimits = 0
+  }
+
+  /**
+   * Retry an operation with exponential backoff
+   * Handles transient network errors like DNS resolution failures and rate limiting
+   * Includes automatic fallback RPC switching on persistent rate limits
   */
  private async retryOperation<T>(
    operation: () => Promise<T>,
-    maxRetries: number = 3,
-    delayMs: number = 2000,
-    operationName: string = 'operation'
+    maxRetries: number,
+    initialDelayMs: number,
+    operationName: string
  ): Promise<T> {
-    let lastError: Error | null = null
+    let lastError: any = null
    
    for (let attempt = 1; attempt <= maxRetries; attempt++) {
      try {
@@ -92,25 +142,49 @@ export class DriftService {
      } catch (error: any) {
        lastError = error
        
-        // Check if it's a transient network error
+        // Check if this is a rate limit error
+        const isRateLimit = 
+          error?.message?.includes('429') ||
+          error?.message?.includes('Too Many Requests') ||
+          error?.message?.includes('compute units per second')
+        
+        // Check if this is a transient error worth retrying
        const isTransient = 
+          isRateLimit ||
          error?.message?.includes('fetch failed') ||
-          error?.message?.includes('EAI_AGAIN') ||
          error?.message?.includes('ENOTFOUND') ||
          error?.message?.includes('ETIMEDOUT') ||
          error?.message?.includes('ECONNREFUSED') ||
          error?.code === 'EAI_AGAIN' ||
          error?.cause?.code === 'EAI_AGAIN'
        
-        console.log(`🔍 Error detection: isTransient=${isTransient}, attempt=${attempt}/${maxRetries}`)
+        console.log(`🔍 Error detection: isTransient=${isTransient}, isRateLimit=${isRateLimit}, attempt=${attempt}/${maxRetries}`)
        console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`)
        
+        // Track consecutive rate limits
+        if (isRateLimit) {
+          this.consecutiveRateLimits++
+          
+          // After 2 consecutive rate limits, try switching to fallback RPC
+          if (this.consecutiveRateLimits >= 2 && this.switchToFallbackRpc()) {
+            console.log('✅ Switched to fallback RPC, retrying immediately...')
+            continue // Retry immediately with fallback
+          }
+        } else {
+          this.consecutiveRateLimits = 0 // Reset on non-rate-limit errors
+        }
+        
        if (!isTransient || attempt === maxRetries) {
          // Non-transient error or max retries reached - fail immediately
          console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`)
          throw error
        }
        
+        // Use longer delays for rate limits (need RPC to recover)
+        const delayMs = isRateLimit 
+          ? initialDelayMs * Math.pow(2, attempt) // Exponential for rate limits: 2s → 4s → 8s
+          : initialDelayMs // Fixed delay for DNS issues
+        
        console.log(`⚠️ ${operationName} failed (attempt ${attempt}/${maxRetries}): ${error?.message || error}`)
        console.log(`⏳ Retrying in ${delayMs}ms...`)
        
@@ -120,9 +194,7 @@ export class DriftService {
    }
    
    throw lastError || new Error(`${operationName} failed after ${maxRetries} retries`)
-  }
-
-  /**
+  }  /**
   * Initialize Drift client and subscribe to account updates
   * Includes automatic retry for transient network failures (DNS, timeouts)
   */
@@ -394,6 +466,7 @@ export function getDriftService(): DriftService {
  if (!driftServiceInstance) {
    const config: DriftConfig = {
      rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
+      fallbackRpcUrl: process.env.SOLANA_FALLBACK_RPC_URL, // Optional fallback
      walletPrivateKey: process.env.DRIFT_WALLET_PRIVATE_KEY || '',
      env: (process.env.DRIFT_ENV as 'mainnet-beta' | 'devnet') || 'mainnet-beta',
    }
--- a/lib/startup/init-position-manager.ts
+++ b/lib/startup/init-position-manager.ts
@@ -52,6 +52,7 @@ async function validateOpenTrades() {
    
    // Get both truly open trades AND recently "closed" trades (last 24h)
    // Recently closed trades might still be open if close transaction failed
+    // TEMPORARILY REDUCED: Check only last 5 closed trades to avoid rate limiting on startup
    const [openTrades, recentlyClosedTrades] = await Promise.all([
      prisma.trade.findMany({
        where: { status: 'open' },
@@ -60,10 +61,10 @@ async function validateOpenTrades() {
      prisma.trade.findMany({
        where: {
          exitReason: { not: null },
-          exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } // Last 24 hours
+          exitTime: { gte: new Date(Date.now() - 6 * 60 * 60 * 1000) } // Last 6 hours (reduced from 24h)
        },
        orderBy: { exitTime: 'desc' },
-        take: 20 // Check last 20 closed trades
+        take: 5 // Reduced from 20 to avoid rate limiting
      })
    ])
    
--- a/lib/trading/position-manager.ts
+++ b/lib/trading/position-manager.ts
@@ -717,36 +717,48 @@ export class PositionManager {
      console.log(`🔒 SL moved to +${this.config.breakEvenTriggerPercent}% (${this.config.takeProfit1SizePercent}% closed, ${100 - this.config.takeProfit1SizePercent}% remaining): ${newStopLossPrice.toFixed(4)}`)
      
      // CRITICAL: Cancel old on-chain SL orders and place new ones at updated price
+      // BUT: Only if this is the ONLY active trade on this symbol
+      // Multiple positions on same symbol = can't distinguish which orders belong to which trade
      try {
-        console.log('🗑️ Cancelling old stop loss orders...')
-        const { cancelAllOrders, placeExitOrders } = await import('../drift/orders')
-        const cancelResult = await cancelAllOrders(trade.symbol)
-        if (cancelResult.success) {
-          console.log(`✅ Cancelled ${cancelResult.cancelledCount || 0} old orders`)
-          
-          // Place new SL orders at breakeven/profit level for remaining position
-          console.log(`🛡️ Placing new SL orders at $${newStopLossPrice.toFixed(4)} for remaining position...`)
-          const exitOrdersResult = await placeExitOrders({
-            symbol: trade.symbol,
-            positionSizeUSD: trade.currentSize,
-            entryPrice: trade.entryPrice,
-            tp1Price: trade.tp2Price, // Only TP2 remains
-            tp2Price: trade.tp2Price, // Dummy, won't be used
-            stopLossPrice: newStopLossPrice,
-            tp1SizePercent: 100, // Close remaining 25% at TP2
-            tp2SizePercent: 0,
-            direction: trade.direction,
-            useDualStops: this.config.useDualStops,
-            softStopPrice: trade.direction === 'long' 
-              ? newStopLossPrice * 1.005  // 0.5% above for long
-              : newStopLossPrice * 0.995, // 0.5% below for short
-            hardStopPrice: newStopLossPrice,
-          })
-          
-          if (exitOrdersResult.success) {
-            console.log('✅ New SL orders placed on-chain at updated price')
-          } else {
-            console.error('❌ Failed to place new SL orders:', exitOrdersResult.error)
+        const otherTradesOnSymbol = Array.from(this.activeTrades.values()).filter(
+          t => t.symbol === trade.symbol && t.id !== trade.id
+        )
+        
+        if (otherTradesOnSymbol.length > 0) {
+          console.log(`⚠️ Multiple trades on ${trade.symbol} detected (${otherTradesOnSymbol.length + 1} total)`)
+          console.log(`⚠️ Skipping order cancellation to avoid wiping other positions' orders`)
+          console.log(`⚠️ Relying on Position Manager software monitoring for remaining ${100 - this.config.takeProfit1SizePercent}%`)
+        } else {
+          console.log('🗑️ Cancelling old stop loss orders...')
+          const { cancelAllOrders, placeExitOrders } = await import('../drift/orders')
+          const cancelResult = await cancelAllOrders(trade.symbol)
+          if (cancelResult.success) {
+            console.log(`✅ Cancelled ${cancelResult.cancelledCount || 0} old orders`)
+            
+            // Place new SL orders at breakeven/profit level for remaining position
+            console.log(`🛡️ Placing new SL orders at $${newStopLossPrice.toFixed(4)} for remaining position...`)
+            const exitOrdersResult = await placeExitOrders({
+              symbol: trade.symbol,
+              positionSizeUSD: trade.currentSize,
+              entryPrice: trade.entryPrice,
+              tp1Price: trade.tp2Price, // Only TP2 remains
+              tp2Price: trade.tp2Price, // Dummy, won't be used
+              stopLossPrice: newStopLossPrice,
+              tp1SizePercent: 100, // Close remaining 25% at TP2
+              tp2SizePercent: 0,
+              direction: trade.direction,
+              useDualStops: this.config.useDualStops,
+              softStopPrice: trade.direction === 'long' 
+                ? newStopLossPrice * 1.005  // 0.5% above for long
+                : newStopLossPrice * 0.995, // 0.5% below for short
+              hardStopPrice: newStopLossPrice,
+            })
+            
+            if (exitOrdersResult.success) {
+              console.log('✅ New SL orders placed on-chain at updated price')
+            } else {
+              console.error('❌ Failed to place new SL orders:', exitOrdersResult.error)
+            }
          }
        }
      } catch (error) {
--- a/scripts/place-manual-orders.mjs
+++ b/scripts/place-manual-orders.mjs
@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+/**
+ * Emergency script to manually place TP/SL orders for unprotected position
+ */
+
+const API_URL = 'http://localhost:3001'
+
+// Trade details from database
+const tradeId = 'cmhyw5pn7000imo07n65ihi8y'
+const symbol = 'SOL-PERP'
+const direction = 'long'
+const entryPrice = 137.069998
+const positionSizeUSD = 42.92421
+const stopLossPrice = 135.69929802
+const tp1Price = 137.618277992
+const tp2Price = 138.029487986
+
+console.log('🚨 EMERGENCY: Placing TP/SL orders manually')
+console.log(`Trade ID: ${tradeId}`)
+console.log(`Symbol: ${symbol}`)
+console.log(`Direction: ${direction}`)
+console.log(`Entry: $${entryPrice}`)
+console.log(`Size: $${positionSizeUSD}`)
+console.log(`SL: $${stopLossPrice}`)
+console.log(`TP1: $${tp1Price}`)
+console.log(`TP2: $${tp2Price}`)
+
+// Call internal API
+const response = await fetch(`${API_URL}/api/trading/execute`, {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+  },
+  body: JSON.stringify({
+    action: 'place_exit_orders_only',
+    symbol,
+    direction,
+    entryPrice,
+    positionSizeUSD,
+    stopLossPrice,
+    tp1Price,
+    tp2Price,
+  })
+})
+
+const result = await response.json()
+console.log('\n📊 Result:', JSON.stringify(result, null, 2))
+
+if (result.success) {
+  console.log('✅ Orders placed successfully!')
+} else {
+  console.error('❌ Failed to place orders:', result.error)
+}