revert: Back to last known working state (27eb5d4)
- Restored Drift client, orders, and .env from commit 27eb5d4
- Updated to current Helius API key
- ISSUE: Execute/check-risk endpoints still hang
- Root cause appears to be Drift SDK initialization hanging at runtime
- Bot initializes successfully at startup but hangs on subsequent Drift calls
- Non-Drift endpoints work fine (settings, positions query)
- Needs investigation: Drift SDK behavior or RPC interaction issue
This commit is contained in:
12
.env
12
.env
@@ -31,15 +31,15 @@ API_SECRET_KEY=2a344f0149442c857fb56c038c0c7d1b113883b830bec792c76f1e0efa15d6bb
|
|||||||
|
|
||||||
# Solana RPC URL (Required for blockchain access)
|
# Solana RPC URL (Required for blockchain access)
|
||||||
#
|
#
|
||||||
# PRIMARY: Helius WebSocket RPC (supports accountSubscribe for Drift SDK)
|
# RECOMMENDED: Helius (best performance, free tier available)
|
||||||
|
# Get free API key at: https://helius.dev
|
||||||
SOLANA_RPC_URL=https://mainnet.helius-rpc.com/?api-key=5e236449-f936-4af7-ae38-f15e2f1a3757
|
SOLANA_RPC_URL=https://mainnet.helius-rpc.com/?api-key=5e236449-f936-4af7-ae38-f15e2f1a3757
|
||||||
|
|
||||||
# Fallback RPC URL (used for trade execution - higher throughput)
|
# Alternative RPC providers (if not using Helius):
|
||||||
# Helius: startup/subscriptions, Alchemy: trade execution
|
#
|
||||||
SOLANA_FALLBACK_RPC_URL=https://solana-mainnet.g.alchemy.com/v2/5A0iA5UYpsmP9gkuezYeg
|
|
||||||
|
|
||||||
# Alternative RPC providers:
|
|
||||||
# QuickNode: https://solana-mainnet.quiknode.pro/YOUR_ENDPOINT/
|
# QuickNode: https://solana-mainnet.quiknode.pro/YOUR_ENDPOINT/
|
||||||
|
# Alchemy: https://solana-mainnet.g.alchemy.com/v2/YOUR_ALCHEMY_KEY
|
||||||
|
# Ankr: https://rpc.ankr.com/solana
|
||||||
# Public (not recommended): https://api.mainnet-beta.solana.com
|
# Public (not recommended): https://api.mainnet-beta.solana.com
|
||||||
|
|
||||||
# ================================
|
# ================================
|
||||||
|
|||||||
@@ -17,44 +17,20 @@ interface ManualWallet {
|
|||||||
|
|
||||||
export interface DriftConfig {
|
export interface DriftConfig {
|
||||||
rpcUrl: string
|
rpcUrl: string
|
||||||
fallbackRpcUrl?: string // Optional fallback RPC for rate limit handling
|
|
||||||
walletPrivateKey: string
|
walletPrivateKey: string
|
||||||
env: 'mainnet-beta' | 'devnet'
|
env: 'mainnet-beta' | 'devnet'
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DriftService {
|
export class DriftService {
|
||||||
private connection: Connection
|
private connection: Connection
|
||||||
private fallbackConnection: Connection | null = null
|
|
||||||
private currentRpcUrl: string
|
|
||||||
private wallet: ManualWallet
|
private wallet: ManualWallet
|
||||||
private keypair: Keypair
|
private keypair: Keypair
|
||||||
private driftClient: DriftClient | null = null
|
private driftClient: DriftClient | null = null
|
||||||
private user: User | null = null
|
private user: User | null = null
|
||||||
private isInitialized: boolean = false
|
private isInitialized: boolean = false
|
||||||
private consecutiveRateLimits: number = 0
|
|
||||||
private usingFallback: boolean = false
|
|
||||||
|
|
||||||
constructor(private config: DriftConfig) {
|
constructor(private config: DriftConfig) {
|
||||||
// SMART STARTUP: Use fallback (Helius) for initialization burst, then switch to primary (Alchemy)
|
|
||||||
// Helius handles the burst better, Alchemy is more stable for ongoing operations
|
|
||||||
const useHeliusForStartup = config.fallbackRpcUrl && !process.env.DISABLE_STARTUP_FALLBACK
|
|
||||||
|
|
||||||
if (useHeliusForStartup) {
|
|
||||||
console.log('🚀 Using fallback RPC for startup initialization (handles bursts better)')
|
|
||||||
this.connection = new Connection(config.fallbackRpcUrl!, 'confirmed')
|
|
||||||
this.currentRpcUrl = config.fallbackRpcUrl!
|
|
||||||
this.usingFallback = true
|
|
||||||
this.fallbackConnection = new Connection(config.rpcUrl, 'confirmed') // Primary becomes the "fallback" temporarily
|
|
||||||
} else {
|
|
||||||
this.connection = new Connection(config.rpcUrl, 'confirmed')
|
this.connection = new Connection(config.rpcUrl, 'confirmed')
|
||||||
this.currentRpcUrl = config.rpcUrl
|
|
||||||
|
|
||||||
// Initialize fallback connection if provided
|
|
||||||
if (config.fallbackRpcUrl) {
|
|
||||||
this.fallbackConnection = new Connection(config.fallbackRpcUrl, 'confirmed')
|
|
||||||
console.log('🔄 Fallback RPC configured:', this.maskRpcUrl(config.fallbackRpcUrl))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create wallet from private key
|
// Create wallet from private key
|
||||||
// Support both formats:
|
// Support both formats:
|
||||||
@@ -100,53 +76,15 @@ export class DriftService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mask RPC URL for logging (hide API key)
|
* Retry helper for handling transient network failures (DNS, timeouts)
|
||||||
*/
|
|
||||||
private maskRpcUrl(url: string): string {
|
|
||||||
return url.replace(/\/v2\/[^/]+/, '/v2/***').replace(/api-key=[^&]+/, 'api-key=***')
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Switch to fallback RPC if available
|
|
||||||
*/
|
|
||||||
private switchToFallbackRpc(): boolean {
|
|
||||||
if (!this.fallbackConnection || this.usingFallback) {
|
|
||||||
return false // No fallback available or already using it
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('🔄 Switching from primary to fallback RPC')
|
|
||||||
this.connection = this.fallbackConnection
|
|
||||||
this.usingFallback = true
|
|
||||||
this.consecutiveRateLimits = 0
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Switch back to primary RPC
|
|
||||||
*/
|
|
||||||
private switchToPrimaryRpc(): void {
|
|
||||||
if (!this.usingFallback) {
|
|
||||||
return // Already using primary
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('🔄 Switching back to primary RPC')
|
|
||||||
this.connection = new Connection(this.config.rpcUrl, 'confirmed')
|
|
||||||
this.usingFallback = false
|
|
||||||
this.consecutiveRateLimits = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Retry an operation with exponential backoff
|
|
||||||
* Handles transient network errors like DNS resolution failures and rate limiting
|
|
||||||
* Includes automatic fallback RPC switching on persistent rate limits
|
|
||||||
*/
|
*/
|
||||||
private async retryOperation<T>(
|
private async retryOperation<T>(
|
||||||
operation: () => Promise<T>,
|
operation: () => Promise<T>,
|
||||||
maxRetries: number,
|
maxRetries: number = 3,
|
||||||
initialDelayMs: number,
|
delayMs: number = 2000,
|
||||||
operationName: string
|
operationName: string = 'operation'
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
let lastError: any = null
|
let lastError: Error | null = null
|
||||||
|
|
||||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||||
try {
|
try {
|
||||||
@@ -154,49 +92,25 @@ export class DriftService {
|
|||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
lastError = error
|
lastError = error
|
||||||
|
|
||||||
// Check if this is a rate limit error
|
// Check if it's a transient network error
|
||||||
const isRateLimit =
|
|
||||||
error?.message?.includes('429') ||
|
|
||||||
error?.message?.includes('Too Many Requests') ||
|
|
||||||
error?.message?.includes('compute units per second')
|
|
||||||
|
|
||||||
// Check if this is a transient error worth retrying
|
|
||||||
const isTransient =
|
const isTransient =
|
||||||
isRateLimit ||
|
|
||||||
error?.message?.includes('fetch failed') ||
|
error?.message?.includes('fetch failed') ||
|
||||||
|
error?.message?.includes('EAI_AGAIN') ||
|
||||||
error?.message?.includes('ENOTFOUND') ||
|
error?.message?.includes('ENOTFOUND') ||
|
||||||
error?.message?.includes('ETIMEDOUT') ||
|
error?.message?.includes('ETIMEDOUT') ||
|
||||||
error?.message?.includes('ECONNREFUSED') ||
|
error?.message?.includes('ECONNREFUSED') ||
|
||||||
error?.code === 'EAI_AGAIN' ||
|
error?.code === 'EAI_AGAIN' ||
|
||||||
error?.cause?.code === 'EAI_AGAIN'
|
error?.cause?.code === 'EAI_AGAIN'
|
||||||
|
|
||||||
console.log(`🔍 Error detection: isTransient=${isTransient}, isRateLimit=${isRateLimit}, attempt=${attempt}/${maxRetries}`)
|
console.log(`🔍 Error detection: isTransient=${isTransient}, attempt=${attempt}/${maxRetries}`)
|
||||||
console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`)
|
console.log(`🔍 Error details: message="${error?.message}", code="${error?.code}", cause.code="${error?.cause?.code}"`)
|
||||||
|
|
||||||
// Track consecutive rate limits
|
|
||||||
if (isRateLimit) {
|
|
||||||
this.consecutiveRateLimits++
|
|
||||||
|
|
||||||
// After 2 consecutive rate limits, try switching to fallback RPC
|
|
||||||
if (this.consecutiveRateLimits >= 2 && this.switchToFallbackRpc()) {
|
|
||||||
console.log('✅ Switched to fallback RPC, retrying immediately...')
|
|
||||||
continue // Retry immediately with fallback
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.consecutiveRateLimits = 0 // Reset on non-rate-limit errors
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isTransient || attempt === maxRetries) {
|
if (!isTransient || attempt === maxRetries) {
|
||||||
// Non-transient error or max retries reached - fail immediately
|
// Non-transient error or max retries reached - fail immediately
|
||||||
console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`)
|
console.log(`❌ Not retrying: isTransient=${isTransient}, maxed=${attempt === maxRetries}`)
|
||||||
throw error
|
throw error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use longer delays for rate limits (need RPC to recover)
|
|
||||||
const delayMs = isRateLimit
|
|
||||||
? initialDelayMs * Math.pow(2, attempt) // Exponential for rate limits: 2s → 4s → 8s
|
|
||||||
: initialDelayMs // Fixed delay for DNS issues
|
|
||||||
|
|
||||||
console.log(`⚠️ ${operationName} failed (attempt ${attempt}/${maxRetries}): ${error?.message || error}`)
|
console.log(`⚠️ ${operationName} failed (attempt ${attempt}/${maxRetries}): ${error?.message || error}`)
|
||||||
console.log(`⏳ Retrying in ${delayMs}ms...`)
|
console.log(`⏳ Retrying in ${delayMs}ms...`)
|
||||||
|
|
||||||
@@ -206,7 +120,9 @@ export class DriftService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw lastError || new Error(`${operationName} failed after ${maxRetries} retries`)
|
throw lastError || new Error(`${operationName} failed after ${maxRetries} retries`)
|
||||||
} /**
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
* Initialize Drift client and subscribe to account updates
|
* Initialize Drift client and subscribe to account updates
|
||||||
* Includes automatic retry for transient network failures (DNS, timeouts)
|
* Includes automatic retry for transient network failures (DNS, timeouts)
|
||||||
*/
|
*/
|
||||||
@@ -247,35 +163,6 @@ export class DriftService {
|
|||||||
this.isInitialized = true
|
this.isInitialized = true
|
||||||
console.log('✅ Drift service initialized successfully')
|
console.log('✅ Drift service initialized successfully')
|
||||||
|
|
||||||
// After successful initialization, switch to primary RPC if we started with fallback
|
|
||||||
if (this.usingFallback && this.fallbackConnection && this.driftClient) {
|
|
||||||
console.log('🔄 Startup complete - switching from Helius to Alchemy for normal operations')
|
|
||||||
|
|
||||||
// Swap connections: primary (Alchemy) becomes active, fallback (Helius) becomes backup
|
|
||||||
const temp = this.connection
|
|
||||||
this.connection = this.fallbackConnection
|
|
||||||
this.fallbackConnection = temp
|
|
||||||
this.usingFallback = false
|
|
||||||
this.consecutiveRateLimits = 0
|
|
||||||
|
|
||||||
// Reinitialize SDK with new connection
|
|
||||||
const sdkConfig = initialize({
|
|
||||||
env: this.config.env === 'devnet' ? 'devnet' : 'mainnet-beta'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Update Drift client's connection reference
|
|
||||||
this.driftClient = new DriftClient({
|
|
||||||
connection: this.connection,
|
|
||||||
wallet: this.wallet as any,
|
|
||||||
programID: new PublicKey(sdkConfig.DRIFT_PROGRAM_ID),
|
|
||||||
opts: { commitment: 'confirmed' }
|
|
||||||
})
|
|
||||||
await this.driftClient.subscribe()
|
|
||||||
this.user = this.driftClient.getUser()
|
|
||||||
|
|
||||||
console.log('✅ Now using Alchemy for stable trading operations')
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('❌ Failed to initialize Drift service after retries:', error)
|
console.error('❌ Failed to initialize Drift service after retries:', error)
|
||||||
throw error
|
throw error
|
||||||
@@ -507,7 +394,6 @@ export function getDriftService(): DriftService {
|
|||||||
if (!driftServiceInstance) {
|
if (!driftServiceInstance) {
|
||||||
const config: DriftConfig = {
|
const config: DriftConfig = {
|
||||||
rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
|
rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
|
||||||
fallbackRpcUrl: process.env.SOLANA_FALLBACK_RPC_URL, // Optional fallback
|
|
||||||
walletPrivateKey: process.env.DRIFT_WALLET_PRIVATE_KEY || '',
|
walletPrivateKey: process.env.DRIFT_WALLET_PRIVATE_KEY || '',
|
||||||
env: (process.env.DRIFT_ENV as 'mainnet-beta' | 'devnet') || 'mainnet-beta',
|
env: (process.env.DRIFT_ENV as 'mainnet-beta' | 'devnet') || 'mainnet-beta',
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -654,7 +654,7 @@ export async function closePosition(
|
|||||||
async function retryWithBackoff<T>(
|
async function retryWithBackoff<T>(
|
||||||
fn: () => Promise<T>,
|
fn: () => Promise<T>,
|
||||||
maxRetries: number = 3,
|
maxRetries: number = 3,
|
||||||
baseDelay: number = 8000 // Increased from 5s to 8s: 8s → 16s → 32s progression for better RPC recovery
|
baseDelay: number = 5000 // Increased from 2s to 5s: 5s → 10s → 20s progression
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user