// Aggressive process cleanup utility import { exec } from 'child_process' import { promisify } from 'util' const execAsync = promisify(exec) class AggressiveCleanup { private static instance: AggressiveCleanup private cleanupInterval: NodeJS.Timeout | null = null private isRunning = false private isInitialized = false private constructor() { // Don't auto-start - let startup.ts control it } static getInstance(): AggressiveCleanup { if (!AggressiveCleanup.instance) { AggressiveCleanup.instance = new AggressiveCleanup() } return AggressiveCleanup.instance } startPeriodicCleanup() { if (this.isInitialized) { console.log('๐Ÿ”„ Aggressive cleanup already initialized') return } this.isInitialized = true console.log('๐Ÿš€ Starting aggressive cleanup system') // In development, use on-demand cleanup instead of periodic if (process.env.NODE_ENV === 'development') { console.log('๐Ÿ”ง Development mode: Using on-demand cleanup (triggered after analysis)') console.log('โœ… On-demand cleanup system ready') return } // Production: Clean up every 10 minutes (longer intervals) this.cleanupInterval = setInterval(async () => { try { await this.cleanupOrphanedProcesses() } catch (error) { console.error('Error in periodic cleanup:', error) } }, 10 * 60 * 1000) // 10 minutes // Also run initial cleanup after 60 seconds setTimeout(() => { this.cleanupOrphanedProcesses().catch(console.error) }, 60000) console.log('โœ… Periodic cleanup system started (10 min intervals)') } async cleanupOrphanedProcesses(): Promise { if (this.isRunning) { console.log('๐Ÿ”’ Cleanup already in progress, skipping...') return } // Check if auto cleanup is disabled (for development) if (process.env.DISABLE_AUTO_CLEANUP === 'true') { console.log('๐Ÿšซ Auto cleanup disabled via DISABLE_AUTO_CLEANUP environment variable') return } this.isRunning = true const isDevelopment = process.env.NODE_ENV === 'development' const cleanupType = isDevelopment ? 'gentle' : 'aggressive' console.log(`๐Ÿงน Running ${cleanupType} cleanup for orphaned processes...`) try { // Check for active analysis sessions try { const { progressTracker } = await import('./progress-tracker') const activeSessions = progressTracker.getActiveSessions() if (activeSessions.length > 0) { console.log(`โš ๏ธ Skipping cleanup - ${activeSessions.length} active analysis sessions detected:`) activeSessions.forEach(session => { const progress = progressTracker.getProgress(session) if (progress) { const activeStep = progress.steps.find(step => step.status === 'active') const currentStep = activeStep ? activeStep.title : 'Unknown' console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`) } else { console.log(` - ${session}: Session info not available`) } }) console.log('โ„น๏ธ Will retry cleanup after analysis completes') return } console.log('โœ… No active analysis sessions detected, proceeding with cleanup') } catch (importError) { console.warn('โš ๏ธ Could not check active sessions, proceeding cautiously with cleanup') console.warn('Import error:', importError) // In case of import errors, be extra cautious - only clean very old processes if (isDevelopment) { console.log('๐Ÿ”ง Development mode with import issues - using aggressive cleanup to clear stuck processes') // In development, if we can't check sessions, assume they're stuck and clean aggressively } } // Find and kill orphaned chromium processes const chromiumProcesses = await this.findChromiumProcesses() if (chromiumProcesses.length > 0) { console.log(`๐Ÿ” Found ${chromiumProcesses.length} chromium processes, evaluating for cleanup...`) // In development, be more selective about which processes to kill let processesToKill = chromiumProcesses if (isDevelopment) { // Only kill processes that are likely orphaned (older than 5 minutes) const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 5 * 60 * 1000) // 5 minutes processesToKill = oldProcesses if (processesToKill.length === 0) { console.log('โœ… All chromium processes appear to be recent and potentially active - skipping cleanup') return } console.log(`๐Ÿ”ง Development mode: Cleaning only ${processesToKill.length} old processes (older than 5 minutes)`) } for (const pid of processesToKill) { try { if (isDevelopment) { // In development, use gentler SIGTERM first console.log(`๐Ÿ”ง Dev mode: Gentle shutdown of process ${pid}`) await execAsync(`kill -TERM ${pid}`) // Give process 3 seconds to shut down gracefully await new Promise(resolve => setTimeout(resolve, 3000)) // Check if process is still running try { await execAsync(`kill -0 ${pid}`) // Process still running, force kill console.log(`โš ๏ธ Process ${pid} didn't shut down gracefully, force killing`) await execAsync(`kill -9 ${pid}`) } catch { // Process already dead, that's good console.log(`โœ… Process ${pid} shut down gracefully`) } } else { // Production: immediate force kill await execAsync(`kill -9 ${pid}`) console.log(`โœ… Killed process ${pid}`) } } catch (error) { // Process might already be dead console.log(`โ„น๏ธ Process ${pid} may already be terminated`) } } } else { console.log('โœ… No orphaned chromium processes found') } // Clean up temp directories try { await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true') console.log('โœ… Cleaned up temp directories') } catch (error) { // Ignore errors } // Clean up shared memory try { await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true') console.log('โœ… Cleaned up shared memory') } catch (error) { // Ignore errors } } catch (error) { console.error(`Error in ${cleanupType} cleanup:`, error) } finally { this.isRunning = false console.log(`๐Ÿ ${cleanupType} cleanup completed`) } } private async findChromiumProcesses(): Promise { try { const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'') return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '') } catch (error) { return [] } } private async filterOldProcesses(pids: string[], maxAgeMs: number): Promise { const oldProcesses: string[] = [] for (const pid of pids) { try { // Get process start time const { stdout } = await execAsync(`ps -o pid,lstart -p ${pid} | tail -1`) const processInfo = stdout.trim() if (processInfo) { // Parse the process start time const parts = processInfo.split(/\s+/) if (parts.length >= 6) { // Format: PID Mon DD HH:MM:SS YYYY const startTimeStr = parts.slice(1).join(' ') const startTime = new Date(startTimeStr) const now = new Date() const processAge = now.getTime() - startTime.getTime() if (processAge > maxAgeMs) { console.log(`๐Ÿ• Process ${pid} is ${Math.round(processAge / 60000)} minutes old - marked for cleanup`) oldProcesses.push(pid) } else { console.log(`๐Ÿ• Process ${pid} is ${Math.round(processAge / 60000)} minutes old - keeping alive`) } } } } catch (error) { // If we can't get process info, assume it's old and safe to clean console.log(`โ“ Could not get age info for process ${pid} - assuming it's old`) oldProcesses.push(pid) } } return oldProcesses } async forceCleanup(): Promise { console.log('๐Ÿšจ Force cleanup initiated...') // Stop periodic cleanup if (this.cleanupInterval) { clearInterval(this.cleanupInterval) } // Run aggressive cleanup await this.cleanupOrphanedProcesses() // Kill all chromium processes try { await execAsync('pkill -9 -f "chromium" 2>/dev/null || true') await execAsync('pkill -9 -f "chrome" 2>/dev/null || true') console.log('โœ… Force killed all browser processes') } catch (error) { console.error('Error in force cleanup:', error) } } // New method for on-demand cleanup after complete automation cycle async runPostAnalysisCleanup(): Promise { // Check if auto cleanup is disabled (for development) if (process.env.DISABLE_AUTO_CLEANUP === 'true') { console.log('๐Ÿšซ Post-analysis cleanup disabled via DISABLE_AUTO_CLEANUP environment variable') return } console.log('๐Ÿงน Post-cycle cleanup triggered (analysis + decision complete)...') // Wait for all browser processes to fully close console.log('โณ Waiting 3 seconds for all processes to close gracefully...') await new Promise(resolve => setTimeout(resolve, 3000)) // Always run cleanup after complete automation cycle - don't check for active sessions // since the analysis is complete and we need to ensure all processes are cleaned up console.log('๐Ÿงน Running comprehensive post-cycle cleanup (ignoring session status)...') try { // Find all chromium processes const chromiumProcesses = await this.findChromiumProcesses() if (chromiumProcesses.length === 0) { console.log('โœ… No chromium processes found to clean up') return } console.log(`๐Ÿ” Found ${chromiumProcesses.length} chromium processes for post-analysis cleanup`) // In post-analysis cleanup, we're more aggressive since analysis is complete // Try graceful shutdown first for (const pid of chromiumProcesses) { try { console.log(`๐Ÿ”ง Attempting graceful shutdown of process ${pid}`) await execAsync(`kill -TERM ${pid}`) } catch (error) { console.log(`โ„น๏ธ Process ${pid} may already be terminated`) } } // Wait for graceful shutdown await new Promise(resolve => setTimeout(resolve, 5000)) // Check which processes are still running and force kill them const stillRunning = await this.findStillRunningProcesses(chromiumProcesses) if (stillRunning.length > 0) { console.log(`๐Ÿ—ก๏ธ Force killing ${stillRunning.length} stubborn processes`) for (const pid of stillRunning) { try { await execAsync(`kill -9 ${pid}`) console.log(`๐Ÿ’€ Force killed process ${pid}`) } catch (error) { console.log(`โ„น๏ธ Process ${pid} already terminated`) } } } else { console.log('โœ… All processes shut down gracefully') } // Clean up temp directories and shared memory try { await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true') await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true') await execAsync('rm -rf /tmp/.org.chromium.* 2>/dev/null || true') console.log('โœ… Cleaned up temporary files and shared memory') } catch (error) { console.error('Warning: Could not clean up temporary files:', error) } console.log('โœ… Post-analysis cleanup completed successfully') } catch (error) { console.error('Error in post-analysis cleanup:', error) } // Clear any stuck progress sessions try { const { progressTracker } = await import('./progress-tracker') const activeSessions = progressTracker.getActiveSessions() if (activeSessions.length > 0) { console.log(`๐Ÿงน Force clearing ${activeSessions.length} potentially stuck sessions`) activeSessions.forEach(session => { console.log(`๐Ÿงน Force clearing session: ${session}`) progressTracker.deleteSession(session) }) } } catch (error) { console.warn('Could not clear progress sessions:', error) } } // Signal that an analysis cycle is complete and all processes should be cleaned up async signalAnalysisCycleComplete(): Promise { console.log('๐ŸŽฏ Analysis cycle completion signal received') // Wait for graceful shutdown of analysis-related processes console.log('โณ Waiting 5 seconds for graceful process shutdown...') await new Promise(resolve => setTimeout(resolve, 5000)) // Check if there are any active progress sessions first const activeSessions = await this.checkActiveAnalysisSessions() if (activeSessions > 0) { console.log(`โš ๏ธ Found ${activeSessions} active analysis sessions, skipping aggressive cleanup`) return } // Only run cleanup if no active sessions console.log('๐Ÿงน No active sessions detected, running post-analysis cleanup...') await this.cleanupPostAnalysisProcesses() } private async checkActiveAnalysisSessions(): Promise { // Check if progress tracker has any active sessions try { // This is a simple check - in a real scenario you might want to check actual session state const { stdout } = await execAsync('pgrep -f "automation-.*-.*" | wc -l') return parseInt(stdout.trim()) || 0 } catch (error) { return 0 } } private async cleanupPostAnalysisProcesses(): Promise { console.log('๐Ÿšจ Post-analysis cleanup - targeting orphaned browser processes') try { // Find all chromium processes const chromiumProcesses = await this.findChromiumProcesses() if (chromiumProcesses.length === 0) { console.log('โœ… No chromium processes found to clean up') return } console.log(`๐Ÿ” Found ${chromiumProcesses.length} chromium processes`) // Filter out processes that are too new (less than 2 minutes old) const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 2 * 60) // 2 minutes if (oldProcesses.length === 0) { console.log('โœ… All chromium processes are recent, not cleaning up') return } console.log(`๐Ÿงน Cleaning up ${oldProcesses.length} old chromium processes`) // Try graceful shutdown first for (const pid of oldProcesses) { try { console.log(`๏ฟฝ Attempting graceful shutdown of process ${pid}`) await execAsync(`kill -TERM ${pid}`) } catch (error) { console.log(`โ„น๏ธ Process ${pid} may already be terminated`) } } // Wait for graceful shutdown await new Promise(resolve => setTimeout(resolve, 3000)) // Check which processes are still running and force kill only those const stillRunning = await this.findStillRunningProcesses(oldProcesses) if (stillRunning.length > 0) { console.log(`๐Ÿ—ก๏ธ Force killing ${stillRunning.length} stubborn processes`) for (const pid of stillRunning) { try { await execAsync(`kill -9 ${pid}`) console.log(`๐Ÿ’€ Force killed process ${pid}`) } catch (error) { console.log(`โ„น๏ธ Process ${pid} already terminated`) } } } console.log('โœ… Post-analysis cleanup completed') } catch (error) { console.error('Error in post-analysis cleanup:', error) } } private async findStillRunningProcesses(pids: string[]): Promise { const stillRunning: string[] = [] for (const pid of pids) { try { await execAsync(`kill -0 ${pid}`) // Check if process exists stillRunning.push(pid) } catch (error) { // Process is already dead } } return stillRunning } // Method to get detailed process information for debugging async getProcessInfo(): Promise { try { console.log('๐Ÿ” Current browser process information:') // Get all chromium processes with detailed info const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep') const processes = stdout.trim().split('\n').filter(line => line.length > 0) if (processes.length === 0) { console.log('โœ… No browser processes currently running') return } console.log(`๐Ÿ“Š Found ${processes.length} browser processes:`) processes.forEach((process, index) => { const parts = process.split(/\s+/) const pid = parts[1] const cpu = parts[2] const mem = parts[3] const command = parts.slice(10).join(' ') console.log(` ${index + 1}. PID: ${pid}, CPU: ${cpu}%, MEM: ${mem}%, CMD: ${command.substring(0, 100)}...`) }) // Get memory usage const { stdout: memInfo } = await execAsync('free -h') console.log('๐Ÿ’พ Memory usage:') console.log(memInfo) } catch (error) { console.error('Error getting process info:', error) } } stop(): void { if (this.cleanupInterval) { clearInterval(this.cleanupInterval) this.cleanupInterval = null } } } // Initialize the aggressive cleanup const aggressiveCleanup = AggressiveCleanup.getInstance() export default aggressiveCleanup