CRITICAL BUG FIX: Cleanup process was interfering with active analysis sessions - Aggressive cleanup was running during active analysis, causing navigation failures - Progress tracking was not properly coordinated with cleanup system - No session state checking before process termination 1. AUTOMATION SERVICE COORDINATION: - Added proper progress tracking to automation cycles - Created unique session IDs for each analysis run - Integrated with progressTracker for session state management - Added post-analysis cleanup triggers with proper timing 2. ENHANCED CLEANUP INTELLIGENCE: - Improved session checking with detailed logging of active sessions - Added process age filtering in development mode (only kill >5min old processes) - Better error handling when progress tracker import fails - More granular cleanup control with session state awareness 3. TIMING IMPROVEMENTS: - Post-analysis cleanup now waits for session completion - Added proper delays between analysis phases - Implemented graceful cleanup deferral when sessions are active - Added delayed cleanup fallback for stuck sessions 4. DEVELOPMENT MODE SAFETY: - Gentler SIGTERM → SIGKILL progression for development - Only clean processes older than 5 minutes during dev - Better logging of process age and cleanup decisions - Safer fallback behavior when session tracking fails This resolves the 'Failed to navigate to layout' errors by ensuring cleanup doesn't interfere with active browser sessions during analysis.
291 lines
10 KiB
TypeScript
291 lines
10 KiB
TypeScript
// Aggressive process cleanup utility
|
||
import { exec } from 'child_process'
|
||
import { promisify } from 'util'
|
||
|
||
const execAsync = promisify(exec)
|
||
|
||
class AggressiveCleanup {
|
||
private static instance: AggressiveCleanup
|
||
private cleanupInterval: NodeJS.Timeout | null = null
|
||
private isRunning = false
|
||
private isInitialized = false
|
||
|
||
private constructor() {
|
||
// Don't auto-start - let startup.ts control it
|
||
}
|
||
|
||
static getInstance(): AggressiveCleanup {
|
||
if (!AggressiveCleanup.instance) {
|
||
AggressiveCleanup.instance = new AggressiveCleanup()
|
||
}
|
||
return AggressiveCleanup.instance
|
||
}
|
||
|
||
startPeriodicCleanup() {
|
||
if (this.isInitialized) {
|
||
console.log('🔄 Aggressive cleanup already initialized')
|
||
return
|
||
}
|
||
|
||
this.isInitialized = true
|
||
console.log('🚀 Starting aggressive cleanup system')
|
||
|
||
// In development, use on-demand cleanup instead of periodic
|
||
if (process.env.NODE_ENV === 'development') {
|
||
console.log('🔧 Development mode: Using on-demand cleanup (triggered after analysis)')
|
||
console.log('✅ On-demand cleanup system ready')
|
||
return
|
||
}
|
||
|
||
// Production: Clean up every 10 minutes (longer intervals)
|
||
this.cleanupInterval = setInterval(async () => {
|
||
try {
|
||
await this.cleanupOrphanedProcesses()
|
||
} catch (error) {
|
||
console.error('Error in periodic cleanup:', error)
|
||
}
|
||
}, 10 * 60 * 1000) // 10 minutes
|
||
|
||
// Also run initial cleanup after 60 seconds
|
||
setTimeout(() => {
|
||
this.cleanupOrphanedProcesses().catch(console.error)
|
||
}, 60000)
|
||
|
||
console.log('✅ Periodic cleanup system started (10 min intervals)')
|
||
}
|
||
|
||
async cleanupOrphanedProcesses(): Promise<void> {
|
||
if (this.isRunning) {
|
||
console.log('🔒 Cleanup already in progress, skipping...')
|
||
return
|
||
}
|
||
|
||
this.isRunning = true
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
|
||
|
||
console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)
|
||
|
||
try {
|
||
// Check for active analysis sessions
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Skipping cleanup - ${activeSessions.length} active analysis sessions detected:`)
|
||
activeSessions.forEach(session => {
|
||
const progress = progressTracker.getProgress(session)
|
||
if (progress) {
|
||
const activeStep = progress.steps.find(step => step.status === 'active')
|
||
const currentStep = activeStep ? activeStep.title : 'Unknown'
|
||
console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
|
||
} else {
|
||
console.log(` - ${session}: Session info not available`)
|
||
}
|
||
})
|
||
console.log('ℹ️ Will retry cleanup after analysis completes')
|
||
return
|
||
}
|
||
|
||
console.log('✅ No active analysis sessions detected, proceeding with cleanup')
|
||
} catch (importError) {
|
||
console.warn('⚠️ Could not check active sessions, proceeding cautiously with cleanup')
|
||
console.warn('Import error:', importError)
|
||
|
||
// In case of import errors, be extra cautious - only clean very old processes
|
||
if (isDevelopment) {
|
||
console.log('🔧 Development mode with import issues - skipping cleanup for safety')
|
||
return
|
||
}
|
||
}
|
||
|
||
// Find and kill orphaned chromium processes
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`🔍 Found ${chromiumProcesses.length} chromium processes, evaluating for cleanup...`)
|
||
|
||
// In development, be more selective about which processes to kill
|
||
let processesToKill = chromiumProcesses
|
||
|
||
if (isDevelopment) {
|
||
// Only kill processes that are likely orphaned (older than 5 minutes)
|
||
const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 5 * 60 * 1000) // 5 minutes
|
||
processesToKill = oldProcesses
|
||
|
||
if (processesToKill.length === 0) {
|
||
console.log('✅ All chromium processes appear to be recent and potentially active - skipping cleanup')
|
||
return
|
||
}
|
||
|
||
console.log(`🔧 Development mode: Cleaning only ${processesToKill.length} old processes (older than 5 minutes)`)
|
||
}
|
||
|
||
for (const pid of processesToKill) {
|
||
try {
|
||
if (isDevelopment) {
|
||
// In development, use gentler SIGTERM first
|
||
console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
// Give process 3 seconds to shut down gracefully
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Check if process is still running
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`)
|
||
// Process still running, force kill
|
||
console.log(`⚠️ Process ${pid} didn't shut down gracefully, force killing`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch {
|
||
// Process already dead, that's good
|
||
console.log(`✅ Process ${pid} shut down gracefully`)
|
||
}
|
||
} else {
|
||
// Production: immediate force kill
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`✅ Killed process ${pid}`)
|
||
}
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
} else {
|
||
console.log('✅ No orphaned chromium processes found')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
// Clean up shared memory
|
||
try {
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up shared memory')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error in ${cleanupType} cleanup:`, error)
|
||
} finally {
|
||
this.isRunning = false
|
||
console.log(`🏁 ${cleanupType} cleanup completed`)
|
||
}
|
||
}
|
||
|
||
private async findChromiumProcesses(): Promise<string[]> {
|
||
try {
|
||
const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
|
||
return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
|
||
} catch (error) {
|
||
return []
|
||
}
|
||
}
|
||
|
||
private async filterOldProcesses(pids: string[], maxAgeMs: number): Promise<string[]> {
|
||
const oldProcesses: string[] = []
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
// Get process start time
|
||
const { stdout } = await execAsync(`ps -o pid,lstart -p ${pid} | tail -1`)
|
||
const processInfo = stdout.trim()
|
||
|
||
if (processInfo) {
|
||
// Parse the process start time
|
||
const parts = processInfo.split(/\s+/)
|
||
if (parts.length >= 6) {
|
||
// Format: PID Mon DD HH:MM:SS YYYY
|
||
const startTimeStr = parts.slice(1).join(' ')
|
||
const startTime = new Date(startTimeStr)
|
||
const now = new Date()
|
||
const processAge = now.getTime() - startTime.getTime()
|
||
|
||
if (processAge > maxAgeMs) {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - marked for cleanup`)
|
||
oldProcesses.push(pid)
|
||
} else {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - keeping alive`)
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
// If we can't get process info, assume it's old and safe to clean
|
||
console.log(`❓ Could not get age info for process ${pid} - assuming it's old`)
|
||
oldProcesses.push(pid)
|
||
}
|
||
}
|
||
|
||
return oldProcesses
|
||
}
|
||
|
||
async forceCleanup(): Promise<void> {
|
||
console.log('🚨 Force cleanup initiated...')
|
||
|
||
// Stop periodic cleanup
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
}
|
||
|
||
// Run aggressive cleanup
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
// Kill all chromium processes
|
||
try {
|
||
await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
|
||
await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
|
||
console.log('✅ Force killed all browser processes')
|
||
} catch (error) {
|
||
console.error('Error in force cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// New method for on-demand cleanup after analysis
|
||
async runPostAnalysisCleanup(): Promise<void> {
|
||
console.log('🧹 Post-analysis cleanup triggered...')
|
||
|
||
// Small delay to ensure analysis processes are fully closed
|
||
console.log('⏳ Waiting 3 seconds for analysis processes to close...')
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Check if there are still active sessions before cleaning
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Post-analysis cleanup: Still ${activeSessions.length} active sessions - deferring cleanup`)
|
||
|
||
// Schedule a delayed cleanup in case sessions don't clear properly
|
||
setTimeout(() => {
|
||
console.log('🕐 Running delayed post-analysis cleanup...')
|
||
this.cleanupOrphanedProcesses()
|
||
}, 10000) // 10 seconds later
|
||
return
|
||
}
|
||
} catch (error) {
|
||
console.warn('⚠️ Could not check active sessions for post-analysis cleanup:', error)
|
||
}
|
||
|
||
await this.cleanupOrphanedProcesses()
|
||
}
|
||
|
||
stop(): void {
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
this.cleanupInterval = null
|
||
}
|
||
}
|
||
}
|
||
|
||
// Initialize the aggressive cleanup
|
||
const aggressiveCleanup = AggressiveCleanup.getInstance()
|
||
|
||
export default aggressiveCleanup
|