700296e664
FIXES: - Enhanced signalAnalysisCycleComplete with more intelligent cleanup logic - Added active session detection to avoid killing processes during analysis - Implemented graceful shutdown (SIGTERM) before force kill (SIGKILL) - Only kills processes older than 2 minutes to avoid disrupting active analysis - Added 10 second delay in runPostCycleCleanup to ensure trading decision is complete - Improved process age filtering to prevent premature cleanup - Cleanup now properly correlates with analysis completion + trading decision - Reduced aggressive kills that were happening during active analysis - Better CPU usage management through smarter process lifecycle - Prevents cleanup from interfering with ongoing analysis work This should significantly reduce the zombie process CPU usage issue by ensuring cleanup only happens when analysis work is truly complete and decisions are finalized.
412 lines
15 KiB
TypeScript
412 lines
15 KiB
TypeScript
// Aggressive process cleanup utility
|
||
import { exec } from 'child_process'
|
||
import { promisify } from 'util'
|
||
|
||
const execAsync = promisify(exec)
|
||
|
||
class AggressiveCleanup {
|
||
private static instance: AggressiveCleanup
|
||
private cleanupInterval: NodeJS.Timeout | null = null
|
||
private isRunning = false
|
||
private isInitialized = false
|
||
|
||
private constructor() {
|
||
// Don't auto-start - let startup.ts control it
|
||
}
|
||
|
||
static getInstance(): AggressiveCleanup {
|
||
if (!AggressiveCleanup.instance) {
|
||
AggressiveCleanup.instance = new AggressiveCleanup()
|
||
}
|
||
return AggressiveCleanup.instance
|
||
}
|
||
|
||
startPeriodicCleanup() {
|
||
if (this.isInitialized) {
|
||
console.log('🔄 Aggressive cleanup already initialized')
|
||
return
|
||
}
|
||
|
||
this.isInitialized = true
|
||
console.log('🚀 Starting aggressive cleanup system')
|
||
|
||
// In development, use on-demand cleanup instead of periodic
|
||
if (process.env.NODE_ENV === 'development') {
|
||
console.log('🔧 Development mode: Using on-demand cleanup (triggered after analysis)')
|
||
console.log('✅ On-demand cleanup system ready')
|
||
return
|
||
}
|
||
|
||
// Production: Clean up every 10 minutes (longer intervals)
|
||
this.cleanupInterval = setInterval(async () => {
|
||
try {
|
||
await this.cleanupOrphanedProcesses()
|
||
} catch (error) {
|
||
console.error('Error in periodic cleanup:', error)
|
||
}
|
||
}, 10 * 60 * 1000) // 10 minutes
|
||
|
||
// Also run initial cleanup after 60 seconds
|
||
setTimeout(() => {
|
||
this.cleanupOrphanedProcesses().catch(console.error)
|
||
}, 60000)
|
||
|
||
console.log('✅ Periodic cleanup system started (10 min intervals)')
|
||
}
|
||
|
||
async cleanupOrphanedProcesses(): Promise<void> {
|
||
if (this.isRunning) {
|
||
console.log('🔒 Cleanup already in progress, skipping...')
|
||
return
|
||
}
|
||
|
||
this.isRunning = true
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
|
||
|
||
console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)
|
||
|
||
try {
|
||
// Check for active analysis sessions
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Skipping cleanup - ${activeSessions.length} active analysis sessions detected:`)
|
||
activeSessions.forEach(session => {
|
||
const progress = progressTracker.getProgress(session)
|
||
if (progress) {
|
||
const activeStep = progress.steps.find(step => step.status === 'active')
|
||
const currentStep = activeStep ? activeStep.title : 'Unknown'
|
||
console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
|
||
} else {
|
||
console.log(` - ${session}: Session info not available`)
|
||
}
|
||
})
|
||
console.log('ℹ️ Will retry cleanup after analysis completes')
|
||
return
|
||
}
|
||
|
||
console.log('✅ No active analysis sessions detected, proceeding with cleanup')
|
||
} catch (importError) {
|
||
console.warn('⚠️ Could not check active sessions, proceeding cautiously with cleanup')
|
||
console.warn('Import error:', importError)
|
||
|
||
// In case of import errors, be extra cautious - only clean very old processes
|
||
if (isDevelopment) {
|
||
console.log('🔧 Development mode with import issues - skipping cleanup for safety')
|
||
return
|
||
}
|
||
}
|
||
|
||
// Find and kill orphaned chromium processes
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`🔍 Found ${chromiumProcesses.length} chromium processes, evaluating for cleanup...`)
|
||
|
||
// In development, be more selective about which processes to kill
|
||
let processesToKill = chromiumProcesses
|
||
|
||
if (isDevelopment) {
|
||
// Only kill processes that are likely orphaned (older than 5 minutes)
|
||
const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 5 * 60 * 1000) // 5 minutes
|
||
processesToKill = oldProcesses
|
||
|
||
if (processesToKill.length === 0) {
|
||
console.log('✅ All chromium processes appear to be recent and potentially active - skipping cleanup')
|
||
return
|
||
}
|
||
|
||
console.log(`🔧 Development mode: Cleaning only ${processesToKill.length} old processes (older than 5 minutes)`)
|
||
}
|
||
|
||
for (const pid of processesToKill) {
|
||
try {
|
||
if (isDevelopment) {
|
||
// In development, use gentler SIGTERM first
|
||
console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
// Give process 3 seconds to shut down gracefully
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Check if process is still running
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`)
|
||
// Process still running, force kill
|
||
console.log(`⚠️ Process ${pid} didn't shut down gracefully, force killing`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch {
|
||
// Process already dead, that's good
|
||
console.log(`✅ Process ${pid} shut down gracefully`)
|
||
}
|
||
} else {
|
||
// Production: immediate force kill
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`✅ Killed process ${pid}`)
|
||
}
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
} else {
|
||
console.log('✅ No orphaned chromium processes found')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
// Clean up shared memory
|
||
try {
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up shared memory')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error in ${cleanupType} cleanup:`, error)
|
||
} finally {
|
||
this.isRunning = false
|
||
console.log(`🏁 ${cleanupType} cleanup completed`)
|
||
}
|
||
}
|
||
|
||
private async findChromiumProcesses(): Promise<string[]> {
|
||
try {
|
||
const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
|
||
return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
|
||
} catch (error) {
|
||
return []
|
||
}
|
||
}
|
||
|
||
private async filterOldProcesses(pids: string[], maxAgeMs: number): Promise<string[]> {
|
||
const oldProcesses: string[] = []
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
// Get process start time
|
||
const { stdout } = await execAsync(`ps -o pid,lstart -p ${pid} | tail -1`)
|
||
const processInfo = stdout.trim()
|
||
|
||
if (processInfo) {
|
||
// Parse the process start time
|
||
const parts = processInfo.split(/\s+/)
|
||
if (parts.length >= 6) {
|
||
// Format: PID Mon DD HH:MM:SS YYYY
|
||
const startTimeStr = parts.slice(1).join(' ')
|
||
const startTime = new Date(startTimeStr)
|
||
const now = new Date()
|
||
const processAge = now.getTime() - startTime.getTime()
|
||
|
||
if (processAge > maxAgeMs) {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - marked for cleanup`)
|
||
oldProcesses.push(pid)
|
||
} else {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - keeping alive`)
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
// If we can't get process info, assume it's old and safe to clean
|
||
console.log(`❓ Could not get age info for process ${pid} - assuming it's old`)
|
||
oldProcesses.push(pid)
|
||
}
|
||
}
|
||
|
||
return oldProcesses
|
||
}
|
||
|
||
async forceCleanup(): Promise<void> {
|
||
console.log('🚨 Force cleanup initiated...')
|
||
|
||
// Stop periodic cleanup
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
}
|
||
|
||
// Run aggressive cleanup
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
// Kill all chromium processes
|
||
try {
|
||
await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
|
||
await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
|
||
console.log('✅ Force killed all browser processes')
|
||
} catch (error) {
|
||
console.error('Error in force cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// New method for on-demand cleanup after complete automation cycle
|
||
async runPostAnalysisCleanup(): Promise<void> {
|
||
console.log('🧹 Post-cycle cleanup triggered (analysis + decision complete)...')
|
||
|
||
// Wait for all browser processes to fully close
|
||
console.log('⏳ Waiting 5 seconds for all processes to close gracefully...')
|
||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||
|
||
// Check if there are still active sessions before cleaning
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Post-cycle cleanup: Still ${activeSessions.length} active sessions detected`)
|
||
activeSessions.forEach(session => {
|
||
const progress = progressTracker.getProgress(session)
|
||
if (progress) {
|
||
const activeStep = progress.steps.find(step => step.status === 'active')
|
||
const currentStep = activeStep ? activeStep.title : 'Unknown'
|
||
console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
|
||
}
|
||
})
|
||
|
||
// Force cleanup anyway since cycle is complete
|
||
console.log('<27> Forcing cleanup - analysis cycle is complete regardless of session status')
|
||
|
||
// Clean up the session tracker entries that might be stuck
|
||
activeSessions.forEach(session => {
|
||
console.log(`🧹 Force clearing stuck session: ${session}`)
|
||
progressTracker.deleteSession(session)
|
||
})
|
||
} else {
|
||
console.log('✅ No active sessions detected - proceeding with post-cycle cleanup')
|
||
}
|
||
} catch (error) {
|
||
console.warn('⚠️ Could not check active sessions for post-cycle cleanup:', error)
|
||
}
|
||
|
||
// Always run cleanup after complete automation cycle
|
||
console.log('🧹 Running comprehensive post-cycle cleanup...')
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
console.log('✅ Post-cycle cleanup completed - all analysis processes should be cleaned up')
|
||
}
|
||
|
||
// Signal that an analysis cycle is complete and all processes should be cleaned up
|
||
async signalAnalysisCycleComplete(): Promise<void> {
|
||
console.log('🎯 Analysis cycle completion signal received')
|
||
|
||
// Wait for graceful shutdown of analysis-related processes
|
||
console.log('⏳ Waiting 5 seconds for graceful process shutdown...')
|
||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||
|
||
// Check if there are any active progress sessions first
|
||
const activeSessions = await this.checkActiveAnalysisSessions()
|
||
if (activeSessions > 0) {
|
||
console.log(`⚠️ Found ${activeSessions} active analysis sessions, skipping aggressive cleanup`)
|
||
return
|
||
}
|
||
|
||
// Only run cleanup if no active sessions
|
||
console.log('🧹 No active sessions detected, running post-analysis cleanup...')
|
||
await this.cleanupPostAnalysisProcesses()
|
||
}
|
||
|
||
private async checkActiveAnalysisSessions(): Promise<number> {
|
||
// Check if progress tracker has any active sessions
|
||
try {
|
||
// This is a simple check - in a real scenario you might want to check actual session state
|
||
const { stdout } = await execAsync('pgrep -f "automation-.*-.*" | wc -l')
|
||
return parseInt(stdout.trim()) || 0
|
||
} catch (error) {
|
||
return 0
|
||
}
|
||
}
|
||
|
||
private async cleanupPostAnalysisProcesses(): Promise<void> {
|
||
console.log('🚨 Post-analysis cleanup - targeting orphaned browser processes')
|
||
|
||
try {
|
||
// Find all chromium processes
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
|
||
if (chromiumProcesses.length === 0) {
|
||
console.log('✅ No chromium processes found to clean up')
|
||
return
|
||
}
|
||
|
||
console.log(`🔍 Found ${chromiumProcesses.length} chromium processes`)
|
||
|
||
// Filter out processes that are too new (less than 2 minutes old)
|
||
const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 2 * 60) // 2 minutes
|
||
|
||
if (oldProcesses.length === 0) {
|
||
console.log('✅ All chromium processes are recent, not cleaning up')
|
||
return
|
||
}
|
||
|
||
console.log(`🧹 Cleaning up ${oldProcesses.length} old chromium processes`)
|
||
|
||
// Try graceful shutdown first
|
||
for (const pid of oldProcesses) {
|
||
try {
|
||
console.log(`<EFBFBD> Attempting graceful shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
} catch (error) {
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
|
||
// Wait for graceful shutdown
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Check which processes are still running and force kill only those
|
||
const stillRunning = await this.findStillRunningProcesses(oldProcesses)
|
||
|
||
if (stillRunning.length > 0) {
|
||
console.log(`🗡️ Force killing ${stillRunning.length} stubborn processes`)
|
||
for (const pid of stillRunning) {
|
||
try {
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`💀 Force killed process ${pid}`)
|
||
} catch (error) {
|
||
console.log(`ℹ️ Process ${pid} already terminated`)
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log('✅ Post-analysis cleanup completed')
|
||
|
||
} catch (error) {
|
||
console.error('Error in post-analysis cleanup:', error)
|
||
}
|
||
}
|
||
|
||
private async findStillRunningProcesses(pids: string[]): Promise<string[]> {
|
||
const stillRunning: string[] = []
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`) // Check if process exists
|
||
stillRunning.push(pid)
|
||
} catch (error) {
|
||
// Process is already dead
|
||
}
|
||
}
|
||
|
||
return stillRunning
|
||
}
|
||
|
||
stop(): void {
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
this.cleanupInterval = null
|
||
}
|
||
}
|
||
}
|
||
|
||
// Initialize the aggressive cleanup
|
||
const aggressiveCleanup = AggressiveCleanup.getInstance()
|
||
|
||
export default aggressiveCleanup
|