feat: Hybrid RPC fallback system (Alchemy → Helius)

- Automatic fallback after 2 consecutive rate limits
- Primary: Alchemy (300M CU/month, stable for normal ops)
- Fallback: Helius (10 req/sec, backup for startup bursts)
- Reduced startup validation: 6h window, 5 trades (was 24h, 20 trades)
- Multi-position safety check (prevents order cancellation conflicts)
- Rate limit-aware retry logic with exponential backoff

Implementation:
- lib/drift/client.ts: Added fallbackConnection, switchToFallbackRpc()
- .env: SOLANA_FALLBACK_RPC_URL configuration
- lib/startup/init-position-manager.ts: Reduced validation scope
- lib/trading/position-manager.ts: Multi-position order protection

Tested: System switched to fallback on startup, Position Manager active
Result: 1 active trade being monitored after automatic RPC switch

.env

@@ -31,16 +31,20 @@ API_SECRET_KEY=2a344f0149442c857fb56c038c0c7d1b113883b830bec792c76f1e0efa15d6bb
 
 # Solana RPC URL (Required for blockchain access)
 # 
-# RECOMMENDED: Helius (best performance, free tier available)
+# CRITICAL: Primary RPC for all trading operations
-# Get free API key at: https://helius.dev
+# Current: Alchemy (300M compute units/month free tier)
 SOLANA_RPC_URL=https://solana-mainnet.g.alchemy.com/v2/5A0iA5UYpsmP9gkuezYeg
 
-# Alternative RPC providers (if not using Helius):
+# Fallback RPC URL (Optional but HIGHLY recommended)
-# 
+# Automatically switches to fallback after 2 consecutive rate limits
-# QuickNode: https://solana-mainnet.quiknode.pro/YOUR_ENDPOINT/
+# Use a different provider than primary for best redundancy
-# Alchemy: https://solana-mainnet.g.alchemy.com/v2/YOUR_ALCHEMY_KEY
+SOLANA_FALLBACK_RPC_URL=https://mainnet.helius-rpc.com/v1/?api-key=dcca4bf0-0b91-4f6a-8d12-1c5a4c1c6e5b
-# Ankr: https://rpc.ankr.com/solana
+
-# Public (not recommended): https://api.mainnet-beta.solana.com
+# RPC Provider Comparison (as of Nov 2025):
+# ✅ Alchemy: 300M CU/month, excellent for primary (CURRENT PRIMARY)
+# ⚠️  Helius: 10 req/sec sustained (free tier), good for fallback only
+# QuickNode: Paid plans, very reliable
+# Ankr/Public: Unreliable, not recommended
 
 # ================================
 # REQUIRED - PYTH NETWORK (Price Feeds)

lib/drift/client.ts

@@ -17,20 +17,32 @@ interface ManualWallet {
 
 export interface DriftConfig {
   rpcUrl: string
+  fallbackRpcUrl?: string  // Optional fallback RPC for rate limit handling
   walletPrivateKey: string
   env: 'mainnet-beta' | 'devnet'
 }
 
 export class DriftService {
   private connection: Connection
+  private fallbackConnection: Connection | null = null
+  private currentRpcUrl: string
   private wallet: ManualWallet
   private keypair: Keypair
   private driftClient: DriftClient | null = null
   private user: User | null = null
   private isInitialized: boolean = false
+  private consecutiveRateLimits: number = 0
+  private usingFallback: boolean = false
 
   constructor(private config: DriftConfig) {
     this.connection = new Connection(config.rpcUrl, 'confirmed')
+    this.currentRpcUrl = config.rpcUrl
+    
+    // Initialize fallback connection if provided
+    if (config.fallbackRpcUrl) {
+      this.fallbackConnection = new Connection(config.fallbackRpcUrl, 'confirmed')
+      console.log('🔄 Fallback RPC configured:', this.maskRpcUrl(config.fallbackRpcUrl))
+    }
     
     // Create wallet from private key
     // Support both formats:
@@ -76,15 +88,53 @@ export class DriftService {
   }
 
   /**
-   * Retry helper for handling transient network failures (DNS, timeouts)
+   * Mask RPC URL for logging (hide API key)
+   */
+  private maskRpcUrl(url: string): string {
+    return url.replace(/\/v2\/[^/]+/, '/v2/***').replace(/api-key=[^&]+/, 'api-key=***')
+  }
+
+  /**
+   * Switch to fallback RPC if available
+   */
+  private switchToFallbackRpc(): boolean {
+    if (!this.fallbackConnection || this.usingFallback) {
+      return false // No fallback available or already using it
+    }
+
+    console.log('🔄 Switching from primary to fallback RPC')
+    this.connection = this.fallbackConnection
+    this.usingFallback = true
+    this.consecutiveRateLimits = 0
+    return true
+  }
+
+  /**
+   * Switch back to primary RPC
+   */
+  private switchToPrimaryRpc(): void {
+    if (!this.usingFallback) {
+      return // Already using primary
+    }
+
+    console.log('🔄 Switching back to primary RPC')
+    this.connection = new Connection(this.config.rpcUrl, 'confirmed')
+    this.usingFallback = false
+    this.consecutiveRateLimits = 0
+  }
+
+  /**
+   * Retry an operation with exponential backoff
+   * Handles transient network errors like DNS resolution failures and rate limiting
+   * Includes automatic fallback RPC switching on persistent rate limits
    */
   private async retryOperation<T>(
     operation: () => Promise<T>,
-    maxRetries: number = 3,
+    maxRetries: number,
-    delayMs: number = 2000,
+    initialDelayMs: number,
-    operationName: string = 'operation'
+    operationName: string
   ): Promise<T> {
-    let lastError: Error | null = null
+    let lastError: any = null
     
     for (let attempt = 1; attempt <= maxRetries; attempt++) {
       try {
@@ -92,25 +142,49 @@ export class DriftService {
       } catch (error: any) {
         lastError = error
         
-        // Check if it's a transient network error
+        // Check if this is a rate limit error
+        const isRateLimit = 
+          error?.message?.includes('429') ||
+          error?.message?.includes('Too Many Requests') ||
+          error?.message?.includes('compute units per second')
+        
+        // Check if this is a transient error worth retrying
         const isTransient = 
+          isRateLimit ||
           error?.message?.includes('fetch failed') ||
-          error?.message?.includes('EAI_AGAIN') ||
           error?.message?.includes('ENOTFOUND') ||
           error?.message?.includes('ETIMEDOUT') ||
           error?.message?.includes('ECONNREFUSED') ||
           error?.code === 'EAI_AGAIN' ||
           error?.cause?.code === 'EAI_AGAIN'
         
-        console.log(`🔍 Error detection: isTransient=${isTransient}, attempt=${attempt}/${maxRetries}`)
+        console.log(`🔍 Error detection: isTransient=${isTransient}, isRateLimit=${isRateLimit}, attempt=${attempt}/${maxRetries}`)
         console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`)
         
+        // Track consecutive rate limits
+        if (isRateLimit) {
+          this.consecutiveRateLimits++
+          
+          // After 2 consecutive rate limits, try switching to fallback RPC
+          if (this.consecutiveRateLimits >= 2 && this.switchToFallbackRpc()) {
+            console.log('✅ Switched to fallback RPC, retrying immediately...')
+            continue // Retry immediately with fallback
+          }
+        } else {
+          this.consecutiveRateLimits = 0 // Reset on non-rate-limit errors
+        }
+        
         if (!isTransient || attempt === maxRetries) {
           // Non-transient error or max retries reached - fail immediately
           console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`)
           throw error
         }
         
+        // Use longer delays for rate limits (need RPC to recover)
+        const delayMs = isRateLimit 
+          ? initialDelayMs * Math.pow(2, attempt) // Exponential for rate limits: 2s → 4s → 8s
+          : initialDelayMs // Fixed delay for DNS issues
+        
         console.log(`⚠️ ${operationName} failed (attempt ${attempt}/${maxRetries}): ${error?.message || error}`)
         console.log(`⏳ Retrying in ${delayMs}ms...`)
         
@@ -120,9 +194,7 @@ export class DriftService {
     }
     
     throw lastError || new Error(`${operationName} failed after ${maxRetries} retries`)
-  }
+  }  /**
-
-  /**
    * Initialize Drift client and subscribe to account updates
    * Includes automatic retry for transient network failures (DNS, timeouts)
    */
@@ -394,6 +466,7 @@ export function getDriftService(): DriftService {
   if (!driftServiceInstance) {
     const config: DriftConfig = {
       rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
+      fallbackRpcUrl: process.env.SOLANA_FALLBACK_RPC_URL, // Optional fallback
       walletPrivateKey: process.env.DRIFT_WALLET_PRIVATE_KEY || '',
       env: (process.env.DRIFT_ENV as 'mainnet-beta' | 'devnet') || 'mainnet-beta',
     }

lib/startup/init-position-manager.ts

@@ -52,6 +52,7 @@ async function validateOpenTrades() {
     
     // Get both truly open trades AND recently "closed" trades (last 24h)
     // Recently closed trades might still be open if close transaction failed
+    // TEMPORARILY REDUCED: Check only last 5 closed trades to avoid rate limiting on startup
     const [openTrades, recentlyClosedTrades] = await Promise.all([
       prisma.trade.findMany({
         where: { status: 'open' },
@@ -60,10 +61,10 @@ async function validateOpenTrades() {
       prisma.trade.findMany({
         where: {
           exitReason: { not: null },
-          exitTime: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } // Last 24 hours
+          exitTime: { gte: new Date(Date.now() - 6 * 60 * 60 * 1000) } // Last 6 hours (reduced from 24h)
         },
         orderBy: { exitTime: 'desc' },
-        take: 20 // Check last 20 closed trades
+        take: 5 // Reduced from 20 to avoid rate limiting
       })
     ])
     

lib/trading/position-manager.ts

@@ -717,7 +717,18 @@ export class PositionManager {
       console.log(`🔒 SL moved to +${this.config.breakEvenTriggerPercent}% (${this.config.takeProfit1SizePercent}% closed, ${100 - this.config.takeProfit1SizePercent}% remaining): ${newStopLossPrice.toFixed(4)}`)
       
       // CRITICAL: Cancel old on-chain SL orders and place new ones at updated price
+      // BUT: Only if this is the ONLY active trade on this symbol
+      // Multiple positions on same symbol = can't distinguish which orders belong to which trade
       try {
+        const otherTradesOnSymbol = Array.from(this.activeTrades.values()).filter(
+          t => t.symbol === trade.symbol && t.id !== trade.id
+        )
+        
+        if (otherTradesOnSymbol.length > 0) {
+          console.log(`⚠️ Multiple trades on ${trade.symbol} detected (${otherTradesOnSymbol.length + 1} total)`)
+          console.log(`⚠️ Skipping order cancellation to avoid wiping other positions' orders`)
+          console.log(`⚠️ Relying on Position Manager software monitoring for remaining ${100 - this.config.takeProfit1SizePercent}%`)
+        } else {
           console.log('🗑️ Cancelling old stop loss orders...')
           const { cancelAllOrders, placeExitOrders } = await import('../drift/orders')
           const cancelResult = await cancelAllOrders(trade.symbol)
@@ -749,6 +760,7 @@ export class PositionManager {
               console.error('❌ Failed to place new SL orders:', exitOrdersResult.error)
             }
           }
+        }
       } catch (error) {
         console.error('❌ Failed to update on-chain SL orders:', error)
         // Don't fail the TP1 exit if SL update fails - software monitoring will handle it

scripts/place-manual-orders.mjs

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+/**
+ * Emergency script to manually place TP/SL orders for unprotected position
+ */
+
+const API_URL = 'http://localhost:3001'
+
+// Trade details from database
+const tradeId = 'cmhyw5pn7000imo07n65ihi8y'
+const symbol = 'SOL-PERP'
+const direction = 'long'
+const entryPrice = 137.069998
+const positionSizeUSD = 42.92421
+const stopLossPrice = 135.69929802
+const tp1Price = 137.618277992
+const tp2Price = 138.029487986
+
+console.log('🚨 EMERGENCY: Placing TP/SL orders manually')
+console.log(`Trade ID: ${tradeId}`)
+console.log(`Symbol: ${symbol}`)
+console.log(`Direction: ${direction}`)
+console.log(`Entry: $${entryPrice}`)
+console.log(`Size: $${positionSizeUSD}`)
+console.log(`SL: $${stopLossPrice}`)
+console.log(`TP1: $${tp1Price}`)
+console.log(`TP2: $${tp2Price}`)
+
+// Call internal API
+const response = await fetch(`${API_URL}/api/trading/execute`, {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+  },
+  body: JSON.stringify({
+    action: 'place_exit_orders_only',
+    symbol,
+    direction,
+    entryPrice,
+    positionSizeUSD,
+    stopLossPrice,
+    tp1Price,
+    tp2Price,
+  })
+})
+
+const result = await response.json()
+console.log('\n📊 Result:', JSON.stringify(result, null, 2))
+
+if (result.success) {
+  console.log('✅ Orders placed successfully!')
+} else {
+  console.error('❌ Failed to place orders:', result.error)
+}