root/trading_bot_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: emergency automation fix - stop runaway trading loops

Browse Source 
- Replace automation service with emergency rate-limited version
- Add 5-minute minimum interval between automation starts
- Implement forced Chromium process cleanup on stop
- Backup broken automation service as .broken file
- Emergency service prevents multiple simultaneous automations
- Fixed 1400+ Chromium process accumulation issue
- Tested and confirmed: rate limiting works, processes stay at 0
This commit is contained in:
mindesbunister
2025-07-24 20:33:20 +02:00
parent ab8fb7c202
commit 1e4f305657
23 changed files with 3837 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
lib/automation-service-simple.ts
View File

@@ -1,3 +1,4 @@
// EMERGENCY RATE LIMITING PATCHconst EMERGENCY_MIN_INTERVAL = 10 * 60 * 1000; // 10 minutes minimumconst EMERGENCY_LAST_RUN = { time: 0 };
import { PrismaClient } from '@prisma/client'
import { aiAnalysisService, AnalysisResult } from './ai-analysis'
import { enhancedScreenshotService } from './enhanced-screenshot-simple'
@@ -46,7 +47,7 @@ export interface AutomationStatus {
export class AutomationService {
private isRunning = false
private config: AutomationConfig | null = null
protected config: AutomationConfig | null = null
private intervalId: NodeJS.Timeout | null = null
private stats = {
totalTrades: 0,
@@ -59,7 +60,10 @@ export class AutomationService {
async startAutomation(config: AutomationConfig): Promise<boolean> {
try {
console.log(`🔧 DEBUG: startAutomation called - isRunning: ${this.isRunning}, config exists: ${!!this.config}`)
if (this.isRunning) {
console.log(`⚠️ DEBUG: Automation already running - rejecting restart attempt`)
throw new Error('Automation is already running')
}
@@ -114,7 +118,8 @@ export class AutomationService {
})
// Start automation cycle
this.startAutomationCycle()
// Start automation cycle (price-based if positions exist, time-based if not)
await this.startAutomationCycle()
// Start price monitoring
await priceMonitorService.startMonitoring()
@@ -150,16 +155,33 @@ export class AutomationService {
}
}
private startAutomationCycle(): void {
private async startAutomationCycle(): Promise<void> {
if (!this.config) return
// Get interval in milliseconds based on timeframe
// Check if we have open positions - if so, only use price-based triggers
const hasPositions = await this.hasOpenPositions()
if (hasPositions) {
console.log(`📊 Open positions detected for ${this.config.symbol} - switching to price-proximity mode only`)
console.log(`🎯 Automation will only trigger on SL/TP approach or critical levels`)
// Don't start time-based cycles when positions exist
// Price monitor events (sl_approach, tp_approach, critical_level) are already set up
return
}
// No positions - start normal time-based automation cycle
const intervalMs = this.getIntervalFromTimeframe(this.config.timeframe)
console.log(`🔄 Starting automation cycle every ${intervalMs/1000} seconds`)
this.intervalId = setInterval(async () => {
this.intervalId = setInterval(async () => { const now = Date.now(); if (now - EMERGENCY_LAST_RUN.time < EMERGENCY_MIN_INTERVAL) { console.log("⏸️ EMERGENCY: Rate limiting active, skipping cycle"); return; } EMERGENCY_LAST_RUN.time = now; const originalFunc = async () => {
if (this.isRunning && this.config) {
// Double-check positions before each cycle
const stillHasPositions = await this.hasOpenPositions()
if (stillHasPositions) {
console.log(`📊 Positions opened during automation - stopping time-based cycles`)
this.stopTimeCycles()
return
}
await this.runAutomationCycle()
}
}, intervalMs)
@@ -168,6 +190,14 @@ export class AutomationService {
this.runAutomationCycle()
}
private stopTimeCycles(): void {
if (this.intervalId) {
clearInterval(this.intervalId)
this.intervalId = null
console.log('⏸️ Time-based automation cycles stopped - now in price-proximity mode only')
}
}
private getIntervalFromTimeframe(timeframe: string): number {
// Check if this is a scalping strategy (multiple short timeframes)
if (this.config?.selectedTimeframes) {
@@ -243,12 +273,21 @@ export class AutomationService {
console.error('Failed to update next scheduled time:', dbError)
}
// Step 1: Check for DCA opportunities on existing positions
const dcaOpportunity = await this.checkForDCAOpportunity()
if (dcaOpportunity.shouldDCA) {
console.log('🔄 DCA opportunity found, executing position scaling')
await this.executeDCA(dcaOpportunity)
await this.runPostCycleCleanup('dca_executed')
// Step 1: Check for open positions first - DON'T analyze if positions exist unless DCA is needed
const hasPositions = await this.hasOpenPositions()
if (hasPositions) {
console.log(`📊 Open position detected for ${this.config.symbol}, checking for DCA only`)
// Only check for DCA opportunities on existing positions
const dcaOpportunity = await this.checkForDCAOpportunity()
if (dcaOpportunity.shouldDCA) {
console.log('🔄 DCA opportunity found, executing position scaling')
await this.executeDCA(dcaOpportunity)
await this.runPostCycleCleanup('dca_executed')
} else {
console.log('📊 Position monitoring only - no new analysis needed')
await this.runPostCycleCleanup('position_monitoring_only')
}
return
}
@@ -261,10 +300,12 @@ export class AutomationService {
// return
// }
// Step 3: Take screenshot and analyze
// Step 3: Take screenshot and analyze with error handling
console.log('📊 Performing analysis...')
const analysisResult = await this.performAnalysis()
if (!analysisResult) {
console.log('❌ Analysis failed, skipping cycle')
console.log(`⏰ Next analysis in ${this.getIntervalFromTimeframe(this.config.timeframe)/1000} seconds`)
// Run cleanup when analysis fails
await this.runPostCycleCleanup('analysis_failed')
return
@@ -352,20 +393,24 @@ export class AutomationService {
// Analyze each timeframe with both AI and DIY layouts
const multiTimeframeResults = await this.analyzeMultiTimeframeWithDualLayouts(symbol, timeframes, sessionId)
if (multiTimeframeResults.length === 0) {
console.log('❌ No multi-timeframe analysis results')
progressTracker.updateStep(sessionId, 'capture', 'error', 'No analysis results captured')
// Check if all analyses failed (browser automation issues)
const validResults = multiTimeframeResults.filter(result => result.analysis !== null)
if (validResults.length === 0) {
console.log('❌ All timeframe analyses failed - likely browser automation failure')
console.log(`⏰ Browser automation issues detected - next analysis in ${this.getIntervalFromTimeframe(this.config!.timeframe)/1000} seconds`)
progressTracker.updateStep(sessionId, 'capture', 'error', 'Browser automation failed - will retry on next cycle')
progressTracker.deleteSession(sessionId)
// Mark analysis as complete to allow cleanup
analysisCompletionFlag.markAnalysisComplete(sessionId)
return null
}
progressTracker.updateStep(sessionId, 'capture', 'completed', `Captured ${multiTimeframeResults.length} timeframe analyses`)
progressTracker.updateStep(sessionId, 'capture', 'completed', `Captured ${validResults.length} timeframe analyses`)
progressTracker.updateStep(sessionId, 'analysis', 'active', 'Processing multi-timeframe results...')
// Process and combine multi-timeframe results
const combinedResult = this.combineMultiTimeframeAnalysis(multiTimeframeResults)
// Process and combine multi-timeframe results using valid results only
const combinedResult = this.combineMultiTimeframeAnalysis(validResults)
if (!combinedResult.analysis) {
console.log('❌ Failed to combine multi-timeframe analysis')
@@ -705,32 +750,33 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
// ✅ NEW: Check if we have SOL position available to sell
private async checkCurrentPosition(): Promise<boolean> {
try {
// Check recent trades to see current position
const recentTrades = await prisma.trade.findMany({
where: {
userId: this.config!.userId,
symbol: this.config!.symbol,
status: 'OPEN'
},
orderBy: { createdAt: 'desc' },
take: 5
// Check actual Drift positions instead of database records
const response = await fetch('http://localhost:3000/api/drift/positions')
if (!response.ok) {
console.error('Failed to fetch Drift positions:', response.statusText)
return false
}
const data = await response.json()
const positions = data.positions || []
// Check if we have any positions for our symbol
const symbolPositions = positions.filter((pos: any) => {
const marketSymbol = pos.marketSymbol || pos.market?.symbol || ''
return marketSymbol.includes(this.config!.symbol.replace('USD', ''))
})
// Count open positions
let netPosition = 0
for (const trade of recentTrades) {
if (trade.side === 'BUY') {
netPosition += trade.amount
} else if (trade.side === 'SELL') {
netPosition -= trade.amount
}
console.log(`🔍 Current ${this.config!.symbol} positions: ${symbolPositions.length}`)
if (symbolPositions.length > 0) {
symbolPositions.forEach((pos: any) => {
console.log(`<EFBFBD> Position: ${pos.marketSymbol} ${pos.side} ${pos.baseAssetAmount} @ $${pos.entryPrice}`)
})
}
console.log(`🔍 Current SOL position: ${netPosition.toFixed(4)} SOL`)
return netPosition > 0.001 // Have at least 0.001 SOL to sell
return symbolPositions.length > 0
} catch (error) {
console.error('❌ Error checking current position:', error)
console.error('❌ Error checking current Drift position:', error)
// If we can't check, default to allowing the trade (fail-safe)
return true
}
@@ -1211,8 +1257,8 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
realTradingAmount: this.config!.tradingAmount,
driftTxId: result.transactionId
}),
// Add AI leverage details in metadata
metadata: JSON.stringify({
// Add AI leverage details in learning data
learningData: JSON.stringify({
aiLeverage: {
calculatedLeverage: decision.leverageUsed,
liquidationPrice: decision.liquidationPrice,
@@ -1280,21 +1326,38 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
/**
* Check if there are any open positions for current symbol
*/
private async hasOpenPositions(): Promise<boolean> {
setTempConfig(config: any): void {
this.config = config as AutomationConfig;
}
clearTempConfig(): void {
this.config = null;
}
async hasOpenPositions(): Promise<boolean> {
if (!this.config) return false
try {
const openPositions = await prisma.trade.findMany({
where: {
userId: this.config.userId,
status: 'open',
symbol: this.config.symbol
}
// Check actual Drift positions instead of database records
const response = await fetch('http://localhost:3000/api/drift/positions')
if (!response.ok) {
console.error('Failed to fetch Drift positions:', response.statusText)
return false
}
const data = await response.json()
const positions = data.positions || []
// Check if there are any positions for our symbol
const symbolPositions = positions.filter((pos: any) => {
const marketSymbol = pos.marketSymbol || pos.market?.symbol || ''
return marketSymbol.includes(this.config!.symbol.replace('USD', ''))
})
return openPositions.length > 0
console.log(`🔍 Found ${symbolPositions.length} open Drift positions for ${this.config.symbol}`)
return symbolPositions.length > 0
} catch (error) {
console.error('Error checking open positions:', error)
console.error('Error checking Drift positions:', error)
return false
}
}
@@ -1324,24 +1387,43 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
this.intervalId = null
}
// Reset config to prevent any residual processes
this.config = null
// Store config reference before clearing it
const configRef = this.config
// Stop price monitoring
// Stop price monitoring with force stop if needed
try {
await priceMonitorService.stopMonitoring()
console.log('📊 Price monitoring stopped')
// Double-check and force stop if still running
setTimeout(() => {
if (priceMonitorService.isMonitoring()) {
console.log('⚠️ Price monitor still running, forcing stop...')
priceMonitorService.stopMonitoring()
}
}, 1000)
} catch (error) {
console.error('Failed to stop price monitoring:', error)
// Force stop via API as fallback
try {
await fetch('http://localhost:3000/api/price-monitor', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ action: 'stop_monitoring' })
})
console.log('📊 Price monitoring force-stopped via API')
} catch (apiError) {
console.error('Failed to force stop price monitoring:', apiError)
}
}
// Update database session status to STOPPED
if (this.config) {
if (configRef) {
await prisma.automationSession.updateMany({
where: {
userId: this.config.userId,
symbol: this.config.symbol,
timeframe: this.config.timeframe,
userId: configRef.userId,
symbol: configRef.symbol,
timeframe: configRef.timeframe,
status: 'ACTIVE'
},
data: {
@@ -1349,8 +1431,10 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
updatedAt: new Date()
}
})
console.log('🛑 Database session status updated to STOPPED')
}
// Reset config AFTER using it for database update
this.config = null
console.log('🛑 Automation stopped')
@@ -1409,8 +1493,38 @@ ${validResults.map(r => `• ${r.timeframe}: ${r.analysis?.recommendation} (${r.
// Auto-restart automation if session exists but not running in memory
if (!isActiveInMemory) {
console.log('🔄 Found active session but automation not running, attempting auto-restart...')
await this.autoRestartFromSession(session)
console.log('🔄 Found active session but automation not running, checking if restart is appropriate...')
// Don't auto-restart if there are open positions unless only DCA is needed
const tempConfig = { userId: session.userId, symbol: session.symbol }
this.config = tempConfig as AutomationConfig // Temporarily set config for position check
const hasPositions = await this.hasOpenPositions()
this.config = null // Clear temp config
if (hasPositions) {
console.log('📊 Open positions detected - preventing auto-restart to avoid unwanted analysis')
console.log('💡 Use manual start to override this safety check if needed')
return {
isActive: false,
mode: session.mode as 'SIMULATION' | 'LIVE',
symbol: session.symbol,
timeframe: session.timeframe,
totalTrades: session.totalTrades,
successfulTrades: session.successfulTrades,
winRate: session.winRate,
totalPnL: session.totalPnL,
lastAnalysis: session.lastAnalysis || undefined,
lastTrade: session.lastTrade || undefined,
nextScheduled: session.nextScheduled || undefined,
errorCount: session.errorCount,
lastError: session.lastError || undefined,
nextAnalysisIn: 0,
analysisInterval: 0
}
} else {
console.log('✅ No open positions - safe to auto-restart automation')
await this.autoRestartFromSession(session)
}
}
// Calculate next analysis timing