fix: emergency automation fix - stop runaway trading loops
- Replace automation service with emergency rate-limited version - Add 5-minute minimum interval between automation starts - Implement forced Chromium process cleanup on stop - Backup broken automation service as .broken file - Emergency service prevents multiple simultaneous automations - Fixed 1400+ Chromium process accumulation issue - Tested and confirmed: rate limiting works, processes stay at 0
This commit is contained in:
mindesbunister
411
lib/automation-service-safe.ts
Normal file
411
lib/automation-service-safe.ts
Normal file
@@ -0,0 +1,411 @@
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { enhancedScreenshotService } from './enhanced-screenshot-robust'
|
||||
import { progressTracker } from './progress-tracker'
|
||||
import { analysisCompletionFlag } from './analysis-completion-flag'
|
||||
import { driftTradingService } from './drift-trading-final'
|
||||
import { automatedCleanupService } from './automated-cleanup-service'
|
||||
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
interface AutomationConfig {
|
||||
userId: string
|
||||
mode: 'SIMULATION' | 'LIVE'
|
||||
symbol: string
|
||||
timeframe: string
|
||||
selectedTimeframes: string[]
|
||||
tradingAmount: number
|
||||
maxLeverage: number
|
||||
stopLossPercent: number
|
||||
takeProfitPercent: number
|
||||
maxDailyTrades: number
|
||||
riskPercentage: number
|
||||
dexProvider: string
|
||||
}
|
||||
|
||||
class SafeAutomationService {
|
||||
private isRunning = false
|
||||
private config: AutomationConfig | null = null
|
||||
private intervalId: NodeJS.Timeout | null = null
|
||||
private lastTradeTime = 0
|
||||
private tradeCount = 0
|
||||
|
||||
// SAFETY LIMITS
|
||||
private readonly MIN_TRADE_INTERVAL = 10 * 60 * 1000 // 10 minutes minimum between trades
|
||||
private readonly MAX_TRADES_PER_HOUR = 2
|
||||
private readonly ANALYSIS_COOLDOWN = 5 * 60 * 1000 // 5 minutes between analyses
|
||||
private readonly MAX_DAILY_TRADES = 6
|
||||
private lastAnalysisTime = 0
|
||||
|
||||
private stats = {
|
||||
totalTrades: 0,
|
||||
successfulTrades: 0,
|
||||
winRate: 0,
|
||||
totalPnL: 0,
|
||||
errorCount: 0,
|
||||
lastError: null as string | null,
|
||||
lastAnalysis: null as string | null,
|
||||
nextScheduled: null as string | null,
|
||||
nextAnalysisIn: 0,
|
||||
analysisInterval: 0,
|
||||
currentCycle: 0
|
||||
}
|
||||
|
||||
async startAutomation(config: AutomationConfig): Promise<{ success: boolean, message?: string }> {
|
||||
try {
|
||||
if (this.isRunning) {
|
||||
return { success: false, message: 'Automation is already running' }
|
||||
}
|
||||
|
||||
// SAFETY CHECK: Rate limiting
|
||||
const now = Date.now()
|
||||
if (now - this.lastAnalysisTime < this.ANALYSIS_COOLDOWN) {
|
||||
const remaining = Math.ceil((this.ANALYSIS_COOLDOWN - (now - this.lastAnalysisTime)) / 1000)
|
||||
return {
|
||||
success: false,
|
||||
message: `Rate limit: Wait ${remaining} seconds before starting automation`
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY CHECK: Check for recent trades
|
||||
const recentTrades = await this.checkRecentTrades()
|
||||
if (recentTrades >= this.MAX_TRADES_PER_HOUR) {
|
||||
return {
|
||||
success: false,
|
||||
message: `Rate limit exceeded: ${recentTrades} trades in last hour (max: ${this.MAX_TRADES_PER_HOUR})`
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY CHECK: Daily trade limit
|
||||
const dailyTrades = await this.checkDailyTrades()
|
||||
if (dailyTrades >= this.MAX_DAILY_TRADES) {
|
||||
return {
|
||||
success: false,
|
||||
message: `Daily limit exceeded: ${dailyTrades} trades today (max: ${this.MAX_DAILY_TRADES})`
|
||||
}
|
||||
}
|
||||
|
||||
this.config = config
|
||||
this.isRunning = true
|
||||
this.lastAnalysisTime = now
|
||||
this.tradeCount = 0
|
||||
|
||||
console.log(`🤖 SAFE: Starting automation for ${config.symbol} in ${config.mode} mode`)
|
||||
console.log(`🛡️ SAFETY: Rate limiting enabled - max ${this.MAX_TRADES_PER_HOUR} trades/hour`)
|
||||
console.log(`⏱️ SAFETY: Minimum ${this.MIN_TRADE_INTERVAL/1000/60} minutes between trades`)
|
||||
|
||||
// Start SAFE automation cycle with longer intervals
|
||||
this.startSafeAutomationCycle()
|
||||
|
||||
return { success: true, message: 'Safe automation started with rate limiting' }
|
||||
} catch (error) {
|
||||
console.error('Failed to start safe automation:', error)
|
||||
this.stats.errorCount++
|
||||
this.stats.lastError = error instanceof Error ? error.message : 'Unknown error'
|
||||
return { success: false, message: `Startup failed: ${error instanceof Error ? error.message : 'Unknown error'}` }
|
||||
}
|
||||
}
|
||||
|
||||
private startSafeAutomationCycle(): void {
|
||||
if (!this.config) return
|
||||
|
||||
// SAFETY: Use much longer intervals (minimum 5 minutes)
|
||||
const baseInterval = this.getIntervalFromTimeframe(this.config.timeframe)
|
||||
const safeInterval = Math.max(baseInterval, this.ANALYSIS_COOLDOWN)
|
||||
|
||||
console.log(`🔄 SAFE: Starting automation cycle every ${safeInterval/1000/60} minutes`)
|
||||
|
||||
this.stats.analysisInterval = safeInterval
|
||||
this.stats.nextScheduled = new Date(Date.now() + safeInterval).toISOString()
|
||||
this.stats.nextAnalysisIn = safeInterval
|
||||
|
||||
this.intervalId = setInterval(async () => {
|
||||
if (this.isRunning && this.config) {
|
||||
const now = Date.now()
|
||||
|
||||
// SAFETY: Check cooldown before each cycle
|
||||
if (now - this.lastAnalysisTime < this.ANALYSIS_COOLDOWN) {
|
||||
console.log(`⏸️ SAFETY: Analysis cooldown active, skipping cycle`)
|
||||
return
|
||||
}
|
||||
|
||||
await this.runSafeAutomationCycle()
|
||||
this.lastAnalysisTime = now
|
||||
this.stats.currentCycle++
|
||||
|
||||
// Update next scheduled time
|
||||
this.stats.nextScheduled = new Date(Date.now() + safeInterval).toISOString()
|
||||
this.stats.nextAnalysisIn = safeInterval
|
||||
}
|
||||
}, safeInterval)
|
||||
|
||||
// Run first cycle after delay to prevent immediate execution
|
||||
const initialDelay = 30000 // 30 seconds
|
||||
setTimeout(() => {
|
||||
if (this.isRunning && this.config) {
|
||||
this.runSafeAutomationCycle()
|
||||
this.lastAnalysisTime = Date.now()
|
||||
this.stats.currentCycle++
|
||||
}
|
||||
}, initialDelay)
|
||||
}
|
||||
|
||||
private async runSafeAutomationCycle(): Promise<void> {
|
||||
if (!this.config) return
|
||||
|
||||
const sessionId = `automation_${Date.now()}`
|
||||
|
||||
try {
|
||||
console.log(`\n🔄 SAFE: Running automation cycle ${this.stats.currentCycle + 1} for ${this.config.symbol}`)
|
||||
|
||||
// SAFETY: Check if we can trade
|
||||
const canTrade = await this.canExecuteTrade()
|
||||
if (!canTrade.allowed) {
|
||||
console.log(`⛔ SAFETY: Trade blocked - ${canTrade.reason}`)
|
||||
return
|
||||
}
|
||||
|
||||
progressTracker.createSession(sessionId, `Safe automation cycle for ${this.config.symbol}`)
|
||||
progressTracker.updateStep(sessionId, 'init', 'active', 'Starting safe analysis...')
|
||||
|
||||
// Perform analysis with enhanced cleanup
|
||||
const analysisResult = await this.performSafeAnalysis(sessionId)
|
||||
|
||||
if (!analysisResult) {
|
||||
console.log('❌ Analysis failed, skipping trade execution')
|
||||
progressTracker.updateStep(sessionId, 'analysis', 'error', 'Analysis failed')
|
||||
return
|
||||
}
|
||||
|
||||
this.stats.lastAnalysis = new Date().toISOString()
|
||||
progressTracker.updateStep(sessionId, 'analysis', 'completed', 'Analysis completed successfully')
|
||||
|
||||
// Execute trade only if analysis is strongly bullish/bearish
|
||||
if (this.shouldExecuteTrade(analysisResult)) {
|
||||
const tradeResult = await this.executeSafeTrade(analysisResult)
|
||||
if (tradeResult?.success) {
|
||||
this.stats.totalTrades++
|
||||
this.stats.successfulTrades++
|
||||
this.stats.winRate = (this.stats.successfulTrades / this.stats.totalTrades) * 100
|
||||
this.lastTradeTime = Date.now()
|
||||
}
|
||||
} else {
|
||||
console.log('📊 Analysis result does not meet execution criteria')
|
||||
}
|
||||
|
||||
progressTracker.updateStep(sessionId, 'complete', 'completed', 'Safe automation cycle completed')
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in safe automation cycle:', error)
|
||||
this.stats.errorCount++
|
||||
this.stats.lastError = error instanceof Error ? error.message : 'Unknown error'
|
||||
progressTracker.updateStep(sessionId, 'analysis', 'error', error instanceof Error ? error.message : 'Unknown error')
|
||||
} finally {
|
||||
// GUARANTEED CLEANUP
|
||||
await this.guaranteedCleanup(sessionId)
|
||||
}
|
||||
}
|
||||
|
||||
private async guaranteedCleanup(sessionId: string): Promise<void> {
|
||||
console.log(`🧹 GUARANTEED: Starting cleanup for session ${sessionId}`)
|
||||
|
||||
try {
|
||||
// Force cleanup with timeout protection
|
||||
const cleanupPromise = automatedCleanupService.performCleanup()
|
||||
const timeoutPromise = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Cleanup timeout')), 10000)
|
||||
)
|
||||
|
||||
await Promise.race([cleanupPromise, timeoutPromise])
|
||||
console.log('✅ GUARANTEED: Cleanup completed successfully')
|
||||
} catch (error) {
|
||||
console.error('⚠️ GUARANTEED: Cleanup failed, forcing manual cleanup', error)
|
||||
|
||||
// Manual fallback cleanup
|
||||
try {
|
||||
const { execSync } = require('child_process')
|
||||
execSync('pkill -f "chrome|chromium" 2>/dev/null || true')
|
||||
console.log('✅ GUARANTEED: Manual cleanup completed')
|
||||
} catch (manualError) {
|
||||
console.error('❌ GUARANTEED: Manual cleanup also failed', manualError)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up progress tracking
|
||||
setTimeout(() => {
|
||||
progressTracker.deleteSession(sessionId)
|
||||
}, 5000)
|
||||
}
|
||||
|
||||
private async canExecuteTrade(): Promise<{ allowed: boolean, reason?: string }> {
|
||||
const now = Date.now()
|
||||
|
||||
// Check time-based cooldown
|
||||
if (now - this.lastTradeTime < this.MIN_TRADE_INTERVAL) {
|
||||
const remaining = Math.ceil((this.MIN_TRADE_INTERVAL - (now - this.lastTradeTime)) / 1000 / 60)
|
||||
return { allowed: false, reason: `Trade cooldown: ${remaining} minutes remaining` }
|
||||
}
|
||||
|
||||
// Check hourly limit
|
||||
const recentTrades = await this.checkRecentTrades()
|
||||
if (recentTrades >= this.MAX_TRADES_PER_HOUR) {
|
||||
return { allowed: false, reason: `Hourly limit reached: ${recentTrades}/${this.MAX_TRADES_PER_HOUR}` }
|
||||
}
|
||||
|
||||
// Check daily limit
|
||||
const dailyTrades = await this.checkDailyTrades()
|
||||
if (dailyTrades >= this.MAX_DAILY_TRADES) {
|
||||
return { allowed: false, reason: `Daily limit reached: ${dailyTrades}/${this.MAX_DAILY_TRADES}` }
|
||||
}
|
||||
|
||||
return { allowed: true }
|
||||
}
|
||||
|
||||
private async checkRecentTrades(): Promise<number> {
|
||||
const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000)
|
||||
try {
|
||||
const count = await prisma.trade.count({
|
||||
where: {
|
||||
createdAt: { gte: oneHourAgo },
|
||||
status: { not: 'CANCELLED' }
|
||||
}
|
||||
})
|
||||
return count
|
||||
} catch (error) {
|
||||
console.error('Error checking recent trades:', error)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private async checkDailyTrades(): Promise<number> {
|
||||
const startOfDay = new Date()
|
||||
startOfDay.setHours(0, 0, 0, 0)
|
||||
|
||||
try {
|
||||
const count = await prisma.trade.count({
|
||||
where: {
|
||||
createdAt: { gte: startOfDay },
|
||||
status: { not: 'CANCELLED' }
|
||||
}
|
||||
})
|
||||
return count
|
||||
} catch (error) {
|
||||
console.error('Error checking daily trades:', error)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private async performSafeAnalysis(sessionId: string): Promise<any> {
|
||||
try {
|
||||
if (!this.config) return null
|
||||
|
||||
progressTracker.updateStep(sessionId, 'analysis', 'active', 'Performing safe screenshot analysis...')
|
||||
|
||||
const analysisResult = await enhancedScreenshotService.captureAndAnalyze({
|
||||
symbol: this.config.symbol,
|
||||
timeframe: this.config.timeframe,
|
||||
layouts: ['ai', 'diy'],
|
||||
analyze: true,
|
||||
sessionId
|
||||
})
|
||||
|
||||
return analysisResult
|
||||
} catch (error) {
|
||||
console.error('Error in safe analysis:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private shouldExecuteTrade(analysisResult: any): boolean {
|
||||
if (!analysisResult?.analysis?.recommendation) return false
|
||||
|
||||
const recommendation = analysisResult.analysis.recommendation.toLowerCase()
|
||||
|
||||
// Only execute on strong signals
|
||||
const strongBullish = recommendation.includes('strong buy') || recommendation.includes('very bullish')
|
||||
const strongBearish = recommendation.includes('strong sell') || recommendation.includes('very bearish')
|
||||
|
||||
return strongBullish || strongBearish
|
||||
}
|
||||
|
||||
private async executeSafeTrade(analysisResult: any): Promise<any> {
|
||||
if (!this.config) return null
|
||||
|
||||
try {
|
||||
console.log('💰 SAFE: Executing trade with enhanced safety checks...')
|
||||
|
||||
const tradeParams = {
|
||||
mode: this.config.mode,
|
||||
symbol: this.config.symbol,
|
||||
amount: this.config.tradingAmount,
|
||||
leverage: this.config.maxLeverage,
|
||||
stopLoss: this.config.stopLossPercent,
|
||||
takeProfit: this.config.takeProfitPercent,
|
||||
analysis: analysisResult.analysis,
|
||||
riskPercentage: this.config.riskPercentage
|
||||
}
|
||||
|
||||
const tradeResult = await driftTradingService.executeTrade(tradeParams)
|
||||
|
||||
if (tradeResult?.success) {
|
||||
console.log(`✅ SAFE: Trade executed successfully`)
|
||||
} else {
|
||||
console.log(`❌ SAFE: Trade execution failed: ${tradeResult?.error}`)
|
||||
}
|
||||
|
||||
return tradeResult
|
||||
} catch (error) {
|
||||
console.error('Error executing safe trade:', error)
|
||||
return { success: false, error: error.message }
|
||||
}
|
||||
}
|
||||
|
||||
async stopAutomation(): Promise<{ success: boolean, message?: string }> {
|
||||
try {
|
||||
this.isRunning = false
|
||||
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId)
|
||||
this.intervalId = null
|
||||
console.log('⛔ SAFE: Automation interval cleared')
|
||||
}
|
||||
|
||||
// Force cleanup
|
||||
await automatedCleanupService.performCleanup()
|
||||
|
||||
this.config = null
|
||||
this.stats.nextAnalysisIn = 0
|
||||
this.stats.nextScheduled = null
|
||||
|
||||
console.log('✅ SAFE: Automation stopped successfully')
|
||||
return { success: true, message: 'Safe automation stopped successfully' }
|
||||
} catch (error) {
|
||||
console.error('Error stopping automation:', error)
|
||||
return { success: false, message: error instanceof Error ? error.message : 'Unknown error' }
|
||||
}
|
||||
}
|
||||
|
||||
getStatus() {
|
||||
return {
|
||||
isActive: this.isRunning,
|
||||
mode: this.config?.mode || 'SIMULATION',
|
||||
symbol: this.config?.symbol || 'SOLUSD',
|
||||
timeframe: this.config?.timeframe || '1h',
|
||||
...this.stats
|
||||
}
|
||||
}
|
||||
|
||||
private getIntervalFromTimeframe(timeframe: string): number {
|
||||
// Much longer intervals for safety
|
||||
const intervals: { [key: string]: number } = {
|
||||
'5': 15 * 60 * 1000, // 15 minutes for 5m timeframe
|
||||
'15': 30 * 60 * 1000, // 30 minutes for 15m timeframe
|
||||
'60': 60 * 60 * 1000, // 1 hour for 1h timeframe
|
||||
'240': 2 * 60 * 60 * 1000, // 2 hours for 4h timeframe
|
||||
'1440': 4 * 60 * 60 * 1000 // 4 hours for 1d timeframe
|
||||
}
|
||||
|
||||
return intervals[timeframe] || 60 * 60 * 1000 // Default 1 hour
|
||||
}
|
||||
}
|
||||
|
||||
export const automationService = new SafeAutomationService()
|
||||
Reference in New Issue
Block a user