feat: Hybrid RPC fallback system (Alchemy → Helius)

- Automatic fallback after 2 consecutive rate limits
- Primary: Alchemy (300M CU/month, stable for normal ops)
- Fallback: Helius (10 req/sec, backup for startup bursts)
- Reduced startup validation: 6h window, 5 trades (was 24h, 20 trades)
- Multi-position safety check (prevents order cancellation conflicts)
- Rate limit-aware retry logic with exponential backoff

Implementation:
- lib/drift/client.ts: Added fallbackConnection, switchToFallbackRpc()
- .env: SOLANA_FALLBACK_RPC_URL configuration
- lib/startup/init-position-manager.ts: Reduced validation scope
- lib/trading/position-manager.ts: Multi-position order protection

Tested: System switched to fallback on startup, Position Manager active
Result: 1 active trade being monitored after automatic RPC switch
This commit is contained in:
mindesbunister
2025-11-14 15:28:07 +01:00
parent d5183514bc
commit 7ff78ee0bd
5 changed files with 193 additions and 50 deletions

View File

@@ -17,20 +17,32 @@ interface ManualWallet {
export interface DriftConfig {
rpcUrl: string
fallbackRpcUrl?: string // Optional fallback RPC for rate limit handling
walletPrivateKey: string
env: 'mainnet-beta' | 'devnet'
}
export class DriftService {
private connection: Connection
private fallbackConnection: Connection | null = null
private currentRpcUrl: string
private wallet: ManualWallet
private keypair: Keypair
private driftClient: DriftClient | null = null
private user: User | null = null
private isInitialized: boolean = false
private consecutiveRateLimits: number = 0
private usingFallback: boolean = false
constructor(private config: DriftConfig) {
this.connection = new Connection(config.rpcUrl, 'confirmed')
this.currentRpcUrl = config.rpcUrl
// Initialize fallback connection if provided
if (config.fallbackRpcUrl) {
this.fallbackConnection = new Connection(config.fallbackRpcUrl, 'confirmed')
console.log('🔄 Fallback RPC configured:', this.maskRpcUrl(config.fallbackRpcUrl))
}
// Create wallet from private key
// Support both formats:
@@ -76,15 +88,53 @@ export class DriftService {
}
/**
* Retry helper for handling transient network failures (DNS, timeouts)
* Mask RPC URL for logging (hide API key)
*/
private maskRpcUrl(url: string): string {
return url.replace(/\/v2\/[^/]+/, '/v2/***').replace(/api-key=[^&]+/, 'api-key=***')
}
/**
* Switch to fallback RPC if available
*/
private switchToFallbackRpc(): boolean {
if (!this.fallbackConnection || this.usingFallback) {
return false // No fallback available or already using it
}
console.log('🔄 Switching from primary to fallback RPC')
this.connection = this.fallbackConnection
this.usingFallback = true
this.consecutiveRateLimits = 0
return true
}
/**
* Switch back to primary RPC
*/
private switchToPrimaryRpc(): void {
if (!this.usingFallback) {
return // Already using primary
}
console.log('🔄 Switching back to primary RPC')
this.connection = new Connection(this.config.rpcUrl, 'confirmed')
this.usingFallback = false
this.consecutiveRateLimits = 0
}
/**
* Retry an operation with exponential backoff
* Handles transient network errors like DNS resolution failures and rate limiting
* Includes automatic fallback RPC switching on persistent rate limits
*/
private async retryOperation<T>(
operation: () => Promise<T>,
maxRetries: number = 3,
delayMs: number = 2000,
operationName: string = 'operation'
maxRetries: number,
initialDelayMs: number,
operationName: string
): Promise<T> {
let lastError: Error | null = null
let lastError: any = null
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
@@ -92,25 +142,49 @@ export class DriftService {
} catch (error: any) {
lastError = error
// Check if it's a transient network error
// Check if this is a rate limit error
const isRateLimit =
error?.message?.includes('429') ||
error?.message?.includes('Too Many Requests') ||
error?.message?.includes('compute units per second')
// Check if this is a transient error worth retrying
const isTransient =
isRateLimit ||
error?.message?.includes('fetch failed') ||
error?.message?.includes('EAI_AGAIN') ||
error?.message?.includes('ENOTFOUND') ||
error?.message?.includes('ETIMEDOUT') ||
error?.message?.includes('ECONNREFUSED') ||
error?.code === 'EAI_AGAIN' ||
error?.cause?.code === 'EAI_AGAIN'
console.log(`🔍 Error detection: isTransient=${isTransient}, attempt=${attempt}/${maxRetries}`)
console.log(`🔍 Error detection: isTransient=${isTransient}, isRateLimit=${isRateLimit}, attempt=${attempt}/${maxRetries}`)
console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`)
// Track consecutive rate limits
if (isRateLimit) {
this.consecutiveRateLimits++
// After 2 consecutive rate limits, try switching to fallback RPC
if (this.consecutiveRateLimits >= 2 && this.switchToFallbackRpc()) {
console.log('✅ Switched to fallback RPC, retrying immediately...')
continue // Retry immediately with fallback
}
} else {
this.consecutiveRateLimits = 0 // Reset on non-rate-limit errors
}
if (!isTransient || attempt === maxRetries) {
// Non-transient error or max retries reached - fail immediately
console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`)
throw error
}
// Use longer delays for rate limits (need RPC to recover)
const delayMs = isRateLimit
? initialDelayMs * Math.pow(2, attempt) // Exponential for rate limits: 2s → 4s → 8s
: initialDelayMs // Fixed delay for DNS issues
console.log(`⚠️ ${operationName} failed (attempt ${attempt}/${maxRetries}): ${error?.message || error}`)
console.log(`⏳ Retrying in ${delayMs}ms...`)
@@ -120,9 +194,7 @@ export class DriftService {
}
throw lastError || new Error(`${operationName} failed after ${maxRetries} retries`)
}
/**
} /**
* Initialize Drift client and subscribe to account updates
* Includes automatic retry for transient network failures (DNS, timeouts)
*/
@@ -394,6 +466,7 @@ export function getDriftService(): DriftService {
if (!driftServiceInstance) {
const config: DriftConfig = {
rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
fallbackRpcUrl: process.env.SOLANA_FALLBACK_RPC_URL, // Optional fallback
walletPrivateKey: process.env.DRIFT_WALLET_PRIVATE_KEY || '',
env: (process.env.DRIFT_ENV as 'mainnet-beta' | 'devnet') || 'mainnet-beta',
}