- Completely disable automatic periodic cleanup in development mode - Disable post-analysis cleanup in development mode - Add multiple checks for active sessions (progress tracker, recent processes, API activity) - Increase graceful shutdown timeout from 3s to 5s - Add conservative fallbacks to prevent premature process termination This prevents the cleanup system from killing browser processes during active analysis, resolving the issue where automation analysis was being interrupted by cleanup.
260 lines
8.5 KiB
TypeScript
260 lines
8.5 KiB
TypeScript
// Aggressive process cleanup utility
|
||
import { exec } from 'child_process'
|
||
import { promisify } from 'util'
|
||
|
||
const execAsync = promisify(exec)
|
||
|
||
class AggressiveCleanup {
|
||
private static instance: AggressiveCleanup
|
||
private cleanupInterval: NodeJS.Timeout | null = null
|
||
private isRunning = false
|
||
private isInitialized = false
|
||
private lastApiCallTime = Date.now()
|
||
|
||
private constructor() {
|
||
// Don't auto-start - let startup.ts control it
|
||
}
|
||
|
||
static getInstance(): AggressiveCleanup {
|
||
if (!AggressiveCleanup.instance) {
|
||
AggressiveCleanup.instance = new AggressiveCleanup()
|
||
}
|
||
return AggressiveCleanup.instance
|
||
}
|
||
|
||
startPeriodicCleanup() {
|
||
if (this.isInitialized) {
|
||
console.log('🔄 Aggressive cleanup already initialized')
|
||
return
|
||
}
|
||
|
||
this.isInitialized = true
|
||
console.log('🚀 Starting aggressive cleanup system')
|
||
|
||
// In development, completely disable automatic cleanup to prevent interference
|
||
if (process.env.NODE_ENV === 'development') {
|
||
console.log('🔧 Development mode: Automatic cleanup DISABLED to prevent analysis interference')
|
||
console.log('💡 Use manual cleanup via runPostAnalysisCleanup() or forceCleanup() when needed')
|
||
console.log('✅ Manual cleanup system ready')
|
||
return
|
||
}
|
||
|
||
// Production: Clean up every 10 minutes (longer intervals)
|
||
this.cleanupInterval = setInterval(async () => {
|
||
try {
|
||
await this.cleanupOrphanedProcesses()
|
||
} catch (error) {
|
||
console.error('Error in periodic cleanup:', error)
|
||
}
|
||
}, 10 * 60 * 1000) // 10 minutes
|
||
|
||
// Also run initial cleanup after 60 seconds
|
||
setTimeout(() => {
|
||
this.cleanupOrphanedProcesses().catch(console.error)
|
||
}, 60000)
|
||
|
||
console.log('✅ Periodic cleanup system started (10 min intervals)')
|
||
}
|
||
|
||
async cleanupOrphanedProcesses(): Promise<void> {
|
||
if (this.isRunning) return
|
||
|
||
this.isRunning = true
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
|
||
|
||
console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)
|
||
|
||
try {
|
||
// Multiple checks for active analysis sessions
|
||
let hasActiveSessions = false
|
||
|
||
// Check 1: Progress tracker
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Found ${activeSessions.length} active progress sessions: ${activeSessions.join(', ')}`)
|
||
hasActiveSessions = true
|
||
}
|
||
} catch (importError) {
|
||
console.log('⚠️ Could not check progress tracker, being conservative')
|
||
hasActiveSessions = true // Be conservative if we can't check
|
||
}
|
||
|
||
// Check 2: Recent browser activity (processes less than 2 minutes old)
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
if (chromiumProcesses.length > 0) {
|
||
const recentProcesses = await this.checkProcessAge(chromiumProcesses)
|
||
if (recentProcesses.length > 0) {
|
||
console.log(`⚠️ Found ${recentProcesses.length} recent browser processes (< 2 min old)`)
|
||
hasActiveSessions = true
|
||
}
|
||
}
|
||
|
||
// Check 3: In development, be extra conservative - only cleanup if no recent API calls
|
||
if (isDevelopment) {
|
||
const lastApiCall = this.getLastApiCallTime()
|
||
const timeSinceLastApi = Date.now() - lastApiCall
|
||
if (timeSinceLastApi < 30000) { // 30 seconds
|
||
console.log(`⚠️ Recent API activity detected (${Math.round(timeSinceLastApi/1000)}s ago), skipping cleanup`)
|
||
hasActiveSessions = true
|
||
}
|
||
}
|
||
|
||
if (hasActiveSessions) {
|
||
console.log(`⚠️ Skipping cleanup - active analysis detected`)
|
||
return
|
||
}
|
||
|
||
console.log('✅ No active analysis sessions detected, proceeding with cleanup')
|
||
|
||
// Find and kill orphaned chromium processes
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`Found ${chromiumProcesses.length} chromium processes, cleaning up...`)
|
||
|
||
for (const pid of chromiumProcesses) {
|
||
try {
|
||
if (isDevelopment) {
|
||
// In development, use gentler SIGTERM first
|
||
console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
// Give process 5 seconds to shut down gracefully (increased from 3)
|
||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||
|
||
// Check if process is still running
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`)
|
||
// Process still running, force kill
|
||
console.log(`⚠️ Process ${pid} didn't shut down gracefully, force killing`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch {
|
||
// Process already dead, that's good
|
||
console.log(`✅ Process ${pid} shut down gracefully`)
|
||
}
|
||
} else {
|
||
// Production: immediate force kill
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`✅ Killed process ${pid}`)
|
||
}
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
} else {
|
||
console.log('✅ No orphaned chromium processes found')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
// Clean up shared memory
|
||
try {
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up shared memory')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error in ${cleanupType} cleanup:`, error)
|
||
} finally {
|
||
this.isRunning = false
|
||
}
|
||
}
|
||
|
||
private async findChromiumProcesses(): Promise<string[]> {
|
||
try {
|
||
const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
|
||
return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
|
||
} catch (error) {
|
||
return []
|
||
}
|
||
}
|
||
|
||
private async checkProcessAge(pids: string[]): Promise<string[]> {
|
||
const recentProcesses: string[] = []
|
||
const twoMinutesAgo = Date.now() - (2 * 60 * 1000)
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
const { stdout } = await execAsync(`ps -o lstart= -p ${pid}`)
|
||
const startTime = new Date(stdout.trim()).getTime()
|
||
if (startTime > twoMinutesAgo) {
|
||
recentProcesses.push(pid)
|
||
}
|
||
} catch (error) {
|
||
// Process might not exist anymore, skip
|
||
}
|
||
}
|
||
|
||
return recentProcesses
|
||
}
|
||
|
||
private getLastApiCallTime(): number {
|
||
return this.lastApiCallTime
|
||
}
|
||
|
||
updateApiCallTime(): void {
|
||
this.lastApiCallTime = Date.now()
|
||
}
|
||
|
||
async forceCleanup(): Promise<void> {
|
||
console.log('🚨 Force cleanup initiated...')
|
||
|
||
// Stop periodic cleanup
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
}
|
||
|
||
// Run aggressive cleanup
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
// Kill all chromium processes
|
||
try {
|
||
await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
|
||
await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
|
||
console.log('✅ Force killed all browser processes')
|
||
} catch (error) {
|
||
console.error('Error in force cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// New method for on-demand cleanup after analysis
|
||
async runPostAnalysisCleanup(): Promise<void> {
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
|
||
if (isDevelopment) {
|
||
console.log('🔧 Development mode: Skipping post-analysis cleanup to prevent interference')
|
||
console.log('💡 Manual cleanup available via forceCleanup() if needed')
|
||
return
|
||
}
|
||
|
||
console.log('🧹 Post-analysis cleanup triggered...')
|
||
|
||
// Small delay to ensure analysis processes are fully closed
|
||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||
|
||
await this.cleanupOrphanedProcesses()
|
||
}
|
||
|
||
stop(): void {
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
this.cleanupInterval = null
|
||
}
|
||
}
|
||
}
|
||
|
||
// Initialize the aggressive cleanup
|
||
const aggressiveCleanup = AggressiveCleanup.getInstance()
|
||
|
||
export default aggressiveCleanup
|