efbec62b68
- Change cleanup message from 'DISABLED' to 'Using on-demand cleanup (triggered after analysis)' - Cleanup is now properly enabled and will run after each analysis cycle - Prevents Chromium process accumulation that was causing memory leaks - Maintains the critical cleanup functionality needed for 24/7 operation This restores the working cleanup system that was accidentally reverted.
344 lines
11 KiB
TypeScript
344 lines
11 KiB
TypeScript
// Aggressive process cleanup utility
|
||
import { exec } from 'child_process'
|
||
import { promisify } from 'util'
|
||
|
||
const execAsync = promisify(exec)
|
||
|
||
class AggressiveCleanup {
|
||
private static instance: AggressiveCleanup
|
||
private cleanupInterval: NodeJS.Timeout | null = null
|
||
private isRunning = false
|
||
private isInitialized = false
|
||
private lastApiCallTime = Date.now()
|
||
|
||
private constructor() {
|
||
// Don't auto-start - let startup.ts control it
|
||
}
|
||
|
||
static getInstance(): AggressiveCleanup {
|
||
if (!AggressiveCleanup.instance) {
|
||
AggressiveCleanup.instance = new AggressiveCleanup()
|
||
}
|
||
return AggressiveCleanup.instance
|
||
}
|
||
|
||
startPeriodicCleanup() {
|
||
if (this.isInitialized) {
|
||
console.log('🔄 Aggressive cleanup already initialized')
|
||
return
|
||
}
|
||
|
||
this.isInitialized = true
|
||
console.log('🚀 Starting aggressive cleanup system')
|
||
|
||
// In development, use on-demand cleanup instead of periodic
|
||
if (process.env.NODE_ENV === 'development') {
|
||
console.log('🔧 Development mode: Using on-demand cleanup (triggered after analysis)')
|
||
console.log('✅ On-demand cleanup system ready')
|
||
return
|
||
}
|
||
|
||
// Production: Clean up every 10 minutes (longer intervals)
|
||
this.cleanupInterval = setInterval(async () => {
|
||
try {
|
||
await this.cleanupOrphanedProcesses()
|
||
} catch (error) {
|
||
console.error('Error in periodic cleanup:', error)
|
||
}
|
||
}, 10 * 60 * 1000) // 10 minutes
|
||
|
||
// Also run initial cleanup after 60 seconds
|
||
setTimeout(() => {
|
||
this.cleanupOrphanedProcesses().catch(console.error)
|
||
}, 60000)
|
||
|
||
console.log('✅ Periodic cleanup system started (10 min intervals)')
|
||
}
|
||
|
||
async cleanupOrphanedProcesses(): Promise<void> {
|
||
if (this.isRunning) return
|
||
|
||
this.isRunning = true
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
|
||
|
||
console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)
|
||
|
||
try {
|
||
// Multiple checks for active analysis sessions
|
||
let hasActiveSessions = false
|
||
|
||
// Check 1: Progress tracker
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Found ${activeSessions.length} active progress sessions: ${activeSessions.join(', ')}`)
|
||
hasActiveSessions = true
|
||
}
|
||
} catch (importError) {
|
||
console.log('⚠️ Could not check progress tracker, being conservative')
|
||
hasActiveSessions = true // Be conservative if we can't check
|
||
}
|
||
|
||
// Check 2: Recent browser activity (processes less than 2 minutes old)
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
if (chromiumProcesses.length > 0) {
|
||
const recentProcesses = await this.checkProcessAge(chromiumProcesses)
|
||
if (recentProcesses.length > 0) {
|
||
console.log(`⚠️ Found ${recentProcesses.length} recent browser processes (< 2 min old)`)
|
||
hasActiveSessions = true
|
||
}
|
||
}
|
||
|
||
// Check 3: In development, be extra conservative - only cleanup if no recent API calls
|
||
if (isDevelopment) {
|
||
const lastApiCall = this.getLastApiCallTime()
|
||
const timeSinceLastApi = Date.now() - lastApiCall
|
||
if (timeSinceLastApi < 30000) { // 30 seconds
|
||
console.log(`⚠️ Recent API activity detected (${Math.round(timeSinceLastApi/1000)}s ago), skipping cleanup`)
|
||
hasActiveSessions = true
|
||
}
|
||
}
|
||
|
||
if (hasActiveSessions) {
|
||
console.log(`⚠️ Skipping cleanup - active analysis detected`)
|
||
return
|
||
}
|
||
|
||
console.log('✅ No active analysis sessions detected, proceeding with cleanup')
|
||
|
||
// Find and kill orphaned chromium processes
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`Found ${chromiumProcesses.length} chromium processes, cleaning up...`)
|
||
|
||
for (const pid of chromiumProcesses) {
|
||
try {
|
||
if (isDevelopment) {
|
||
// In development, use gentler SIGTERM first
|
||
console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
// Give process 5 seconds to shut down gracefully (increased from 3)
|
||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||
|
||
// Check if process is still running
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`)
|
||
// Process still running, force kill
|
||
console.log(`⚠️ Process ${pid} didn't shut down gracefully, force killing`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch {
|
||
// Process already dead, that's good
|
||
console.log(`✅ Process ${pid} shut down gracefully`)
|
||
}
|
||
} else {
|
||
// Production: immediate force kill
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`✅ Killed process ${pid}`)
|
||
}
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
} else {
|
||
console.log('✅ No orphaned chromium processes found')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
// Clean up shared memory
|
||
try {
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up shared memory')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error in ${cleanupType} cleanup:`, error)
|
||
} finally {
|
||
this.isRunning = false
|
||
}
|
||
}
|
||
|
||
private async findChromiumProcesses(): Promise<string[]> {
|
||
try {
|
||
const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
|
||
return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
|
||
} catch (error) {
|
||
return []
|
||
}
|
||
}
|
||
|
||
private async checkProcessAge(pids: string[]): Promise<string[]> {
|
||
const recentProcesses: string[] = []
|
||
const twoMinutesAgo = Date.now() - (2 * 60 * 1000)
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
const { stdout } = await execAsync(`ps -o lstart= -p ${pid}`)
|
||
const startTime = new Date(stdout.trim()).getTime()
|
||
if (startTime > twoMinutesAgo) {
|
||
recentProcesses.push(pid)
|
||
}
|
||
} catch (error) {
|
||
// Process might not exist anymore, skip
|
||
}
|
||
}
|
||
|
||
return recentProcesses
|
||
}
|
||
|
||
private getLastApiCallTime(): number {
|
||
return this.lastApiCallTime
|
||
}
|
||
|
||
updateApiCallTime(): void {
|
||
this.lastApiCallTime = Date.now()
|
||
}
|
||
|
||
async forceCleanup(): Promise<void> {
|
||
console.log('🚨 Force cleanup initiated...')
|
||
|
||
// Stop periodic cleanup
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
}
|
||
|
||
// Run aggressive cleanup
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
// Kill all chromium processes
|
||
try {
|
||
await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
|
||
await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
|
||
console.log('✅ Force killed all browser processes')
|
||
} catch (error) {
|
||
console.error('Error in force cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// New method for on-demand cleanup after analysis
|
||
async runPostAnalysisCleanup(): Promise<void> {
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
|
||
if (isDevelopment) {
|
||
console.log('🔧 Development mode: Checking if safe to cleanup...')
|
||
|
||
// In development, still check for completion flags before cleanup
|
||
const isAnalysisComplete = await this.checkAnalysisCompletion()
|
||
if (isAnalysisComplete) {
|
||
console.log('✅ Analysis complete, safe to cleanup Chromium processes')
|
||
await this.cleanupChromeProcessesOnly()
|
||
} else {
|
||
console.log('⏳ Analysis still running, skipping cleanup to prevent interference')
|
||
}
|
||
return
|
||
}
|
||
|
||
console.log('🧹 Post-analysis cleanup triggered...')
|
||
|
||
// Small delay to ensure analysis processes are fully closed
|
||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||
|
||
await this.cleanupOrphanedProcesses()
|
||
}
|
||
|
||
private async checkAnalysisCompletion(): Promise<boolean> {
|
||
try {
|
||
// Check if analysis completion flag exists
|
||
const { analysisCompletionFlag } = await import('./analysis-completion-flag')
|
||
return analysisCompletionFlag.canCleanup()
|
||
} catch (error) {
|
||
console.log('⚠️ Could not check analysis completion, assuming it is complete')
|
||
return true // Assume complete if we can't check
|
||
}
|
||
}
|
||
|
||
private async cleanupChromeProcessesOnly(): Promise<void> {
|
||
console.log('🧹 Cleaning up Chromium processes only...')
|
||
|
||
try {
|
||
// First pass - graceful termination
|
||
const gracefulCommands = [
|
||
"pkill -TERM -f 'chromium.*--remote-debugging-port' || true",
|
||
"pkill -TERM -f 'chromium.*--user-data-dir' || true",
|
||
"pkill -TERM -f '/usr/lib/chromium/chromium' || true",
|
||
"pkill -TERM -f chrome || true"
|
||
]
|
||
|
||
for (const cmd of gracefulCommands) {
|
||
try {
|
||
await execAsync(cmd)
|
||
} catch (error) {
|
||
// Ignore errors - processes might not exist
|
||
}
|
||
}
|
||
|
||
// Wait 2 seconds for graceful shutdown
|
||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||
|
||
// Second pass - force kill
|
||
const forceCommands = [
|
||
"pkill -9 -f 'chromium.*--remote-debugging-port' || true",
|
||
"pkill -9 -f 'chromium.*--user-data-dir' || true",
|
||
"pkill -9 -f '/usr/lib/chromium/chromium' || true",
|
||
"pkill -9 -f chrome || true",
|
||
"pkill -9 -f 'type=zygote' || true",
|
||
"pkill -9 -f 'type=gpu-process' || true",
|
||
"pkill -9 -f 'type=utility' || true",
|
||
"pkill -9 -f 'defunct' || true"
|
||
]
|
||
|
||
for (const cmd of forceCommands) {
|
||
try {
|
||
await execAsync(cmd)
|
||
} catch (error) {
|
||
// Ignore errors - processes might not exist
|
||
}
|
||
}
|
||
|
||
console.log('✅ Chromium processes cleanup completed')
|
||
} catch (error) {
|
||
console.error('❌ Error in Chromium cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// Force cleanup after successful trade execution
|
||
async forceCleanupAfterTrade(): Promise<void> {
|
||
console.log('💰 Trade executed - forcing cleanup of Chromium processes')
|
||
|
||
// Wait longer to ensure analysis is completely done and no new analysis starts
|
||
await new Promise(resolve => setTimeout(resolve, 10000)) // 10 seconds
|
||
|
||
// Check if analysis is still running
|
||
const analysisComplete = await this.checkAnalysisCompletion()
|
||
if (analysisComplete) {
|
||
console.log('✅ Analysis confirmed complete, proceeding with cleanup')
|
||
await this.cleanupChromeProcessesOnly()
|
||
} else {
|
||
console.log('⏳ Analysis still active, skipping cleanup to prevent interference')
|
||
}
|
||
}
|
||
|
||
stop(): void {
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
this.cleanupInterval = null
|
||
}
|
||
}
|
||
}
|
||
|
||
// Initialize the aggressive cleanup
|
||
const aggressiveCleanup = AggressiveCleanup.getInstance()
|
||
|
||
export default aggressiveCleanup
|