cca7303b47
- Moved cleanup trigger from analysis phase to complete automation cycle - Cleanup now runs after trading decision is made (enter/hold/exit) - Added comprehensive post-cycle cleanup that waits for graceful shutdown - Enhanced cleanup coordination with analysis cycle completion signals - Force cleanup after complete cycle to ensure all zombie processes are killed - Added cleanup triggers for all cycle outcomes (trade executed, no opportunity, error, etc.) - Improved timing to wait for browser processes to close properly - Better correlation between analysis completion and process cleanup
360 lines
13 KiB
TypeScript
360 lines
13 KiB
TypeScript
// Aggressive process cleanup utility
|
||
import { exec } from 'child_process'
|
||
import { promisify } from 'util'
|
||
|
||
const execAsync = promisify(exec)
|
||
|
||
class AggressiveCleanup {
|
||
private static instance: AggressiveCleanup
|
||
private cleanupInterval: NodeJS.Timeout | null = null
|
||
private isRunning = false
|
||
private isInitialized = false
|
||
|
||
private constructor() {
|
||
// Don't auto-start - let startup.ts control it
|
||
}
|
||
|
||
static getInstance(): AggressiveCleanup {
|
||
if (!AggressiveCleanup.instance) {
|
||
AggressiveCleanup.instance = new AggressiveCleanup()
|
||
}
|
||
return AggressiveCleanup.instance
|
||
}
|
||
|
||
startPeriodicCleanup() {
|
||
if (this.isInitialized) {
|
||
console.log('🔄 Aggressive cleanup already initialized')
|
||
return
|
||
}
|
||
|
||
this.isInitialized = true
|
||
console.log('🚀 Starting aggressive cleanup system')
|
||
|
||
// In development, use on-demand cleanup instead of periodic
|
||
if (process.env.NODE_ENV === 'development') {
|
||
console.log('🔧 Development mode: Using on-demand cleanup (triggered after analysis)')
|
||
console.log('✅ On-demand cleanup system ready')
|
||
return
|
||
}
|
||
|
||
// Production: Clean up every 10 minutes (longer intervals)
|
||
this.cleanupInterval = setInterval(async () => {
|
||
try {
|
||
await this.cleanupOrphanedProcesses()
|
||
} catch (error) {
|
||
console.error('Error in periodic cleanup:', error)
|
||
}
|
||
}, 10 * 60 * 1000) // 10 minutes
|
||
|
||
// Also run initial cleanup after 60 seconds
|
||
setTimeout(() => {
|
||
this.cleanupOrphanedProcesses().catch(console.error)
|
||
}, 60000)
|
||
|
||
console.log('✅ Periodic cleanup system started (10 min intervals)')
|
||
}
|
||
|
||
async cleanupOrphanedProcesses(): Promise<void> {
|
||
if (this.isRunning) {
|
||
console.log('🔒 Cleanup already in progress, skipping...')
|
||
return
|
||
}
|
||
|
||
this.isRunning = true
|
||
const isDevelopment = process.env.NODE_ENV === 'development'
|
||
const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
|
||
|
||
console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)
|
||
|
||
try {
|
||
// Check for active analysis sessions
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Skipping cleanup - ${activeSessions.length} active analysis sessions detected:`)
|
||
activeSessions.forEach(session => {
|
||
const progress = progressTracker.getProgress(session)
|
||
if (progress) {
|
||
const activeStep = progress.steps.find(step => step.status === 'active')
|
||
const currentStep = activeStep ? activeStep.title : 'Unknown'
|
||
console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
|
||
} else {
|
||
console.log(` - ${session}: Session info not available`)
|
||
}
|
||
})
|
||
console.log('ℹ️ Will retry cleanup after analysis completes')
|
||
return
|
||
}
|
||
|
||
console.log('✅ No active analysis sessions detected, proceeding with cleanup')
|
||
} catch (importError) {
|
||
console.warn('⚠️ Could not check active sessions, proceeding cautiously with cleanup')
|
||
console.warn('Import error:', importError)
|
||
|
||
// In case of import errors, be extra cautious - only clean very old processes
|
||
if (isDevelopment) {
|
||
console.log('🔧 Development mode with import issues - skipping cleanup for safety')
|
||
return
|
||
}
|
||
}
|
||
|
||
// Find and kill orphaned chromium processes
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`🔍 Found ${chromiumProcesses.length} chromium processes, evaluating for cleanup...`)
|
||
|
||
// In development, be more selective about which processes to kill
|
||
let processesToKill = chromiumProcesses
|
||
|
||
if (isDevelopment) {
|
||
// Only kill processes that are likely orphaned (older than 5 minutes)
|
||
const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 5 * 60 * 1000) // 5 minutes
|
||
processesToKill = oldProcesses
|
||
|
||
if (processesToKill.length === 0) {
|
||
console.log('✅ All chromium processes appear to be recent and potentially active - skipping cleanup')
|
||
return
|
||
}
|
||
|
||
console.log(`🔧 Development mode: Cleaning only ${processesToKill.length} old processes (older than 5 minutes)`)
|
||
}
|
||
|
||
for (const pid of processesToKill) {
|
||
try {
|
||
if (isDevelopment) {
|
||
// In development, use gentler SIGTERM first
|
||
console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
|
||
await execAsync(`kill -TERM ${pid}`)
|
||
// Give process 3 seconds to shut down gracefully
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Check if process is still running
|
||
try {
|
||
await execAsync(`kill -0 ${pid}`)
|
||
// Process still running, force kill
|
||
console.log(`⚠️ Process ${pid} didn't shut down gracefully, force killing`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch {
|
||
// Process already dead, that's good
|
||
console.log(`✅ Process ${pid} shut down gracefully`)
|
||
}
|
||
} else {
|
||
// Production: immediate force kill
|
||
await execAsync(`kill -9 ${pid}`)
|
||
console.log(`✅ Killed process ${pid}`)
|
||
}
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
} else {
|
||
console.log('✅ No orphaned chromium processes found')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
// Clean up shared memory
|
||
try {
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up shared memory')
|
||
} catch (error) {
|
||
// Ignore errors
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error in ${cleanupType} cleanup:`, error)
|
||
} finally {
|
||
this.isRunning = false
|
||
console.log(`🏁 ${cleanupType} cleanup completed`)
|
||
}
|
||
}
|
||
|
||
private async findChromiumProcesses(): Promise<string[]> {
|
||
try {
|
||
const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
|
||
return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
|
||
} catch (error) {
|
||
return []
|
||
}
|
||
}
|
||
|
||
private async filterOldProcesses(pids: string[], maxAgeMs: number): Promise<string[]> {
|
||
const oldProcesses: string[] = []
|
||
|
||
for (const pid of pids) {
|
||
try {
|
||
// Get process start time
|
||
const { stdout } = await execAsync(`ps -o pid,lstart -p ${pid} | tail -1`)
|
||
const processInfo = stdout.trim()
|
||
|
||
if (processInfo) {
|
||
// Parse the process start time
|
||
const parts = processInfo.split(/\s+/)
|
||
if (parts.length >= 6) {
|
||
// Format: PID Mon DD HH:MM:SS YYYY
|
||
const startTimeStr = parts.slice(1).join(' ')
|
||
const startTime = new Date(startTimeStr)
|
||
const now = new Date()
|
||
const processAge = now.getTime() - startTime.getTime()
|
||
|
||
if (processAge > maxAgeMs) {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - marked for cleanup`)
|
||
oldProcesses.push(pid)
|
||
} else {
|
||
console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - keeping alive`)
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
// If we can't get process info, assume it's old and safe to clean
|
||
console.log(`❓ Could not get age info for process ${pid} - assuming it's old`)
|
||
oldProcesses.push(pid)
|
||
}
|
||
}
|
||
|
||
return oldProcesses
|
||
}
|
||
|
||
async forceCleanup(): Promise<void> {
|
||
console.log('🚨 Force cleanup initiated...')
|
||
|
||
// Stop periodic cleanup
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
}
|
||
|
||
// Run aggressive cleanup
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
// Kill all chromium processes
|
||
try {
|
||
await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
|
||
await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
|
||
console.log('✅ Force killed all browser processes')
|
||
} catch (error) {
|
||
console.error('Error in force cleanup:', error)
|
||
}
|
||
}
|
||
|
||
// New method for on-demand cleanup after complete automation cycle
|
||
async runPostAnalysisCleanup(): Promise<void> {
|
||
console.log('🧹 Post-cycle cleanup triggered (analysis + decision complete)...')
|
||
|
||
// Wait for all browser processes to fully close
|
||
console.log('⏳ Waiting 5 seconds for all processes to close gracefully...')
|
||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||
|
||
// Check if there are still active sessions before cleaning
|
||
try {
|
||
const { progressTracker } = await import('./progress-tracker')
|
||
const activeSessions = progressTracker.getActiveSessions()
|
||
|
||
if (activeSessions.length > 0) {
|
||
console.log(`⚠️ Post-cycle cleanup: Still ${activeSessions.length} active sessions detected`)
|
||
activeSessions.forEach(session => {
|
||
const progress = progressTracker.getProgress(session)
|
||
if (progress) {
|
||
const activeStep = progress.steps.find(step => step.status === 'active')
|
||
const currentStep = activeStep ? activeStep.title : 'Unknown'
|
||
console.log(` - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
|
||
}
|
||
})
|
||
|
||
// Force cleanup anyway since cycle is complete
|
||
console.log('<27> Forcing cleanup - analysis cycle is complete regardless of session status')
|
||
|
||
// Clean up the session tracker entries that might be stuck
|
||
activeSessions.forEach(session => {
|
||
console.log(`🧹 Force clearing stuck session: ${session}`)
|
||
progressTracker.deleteSession(session)
|
||
})
|
||
} else {
|
||
console.log('✅ No active sessions detected - proceeding with post-cycle cleanup')
|
||
}
|
||
} catch (error) {
|
||
console.warn('⚠️ Could not check active sessions for post-cycle cleanup:', error)
|
||
}
|
||
|
||
// Always run cleanup after complete automation cycle
|
||
console.log('🧹 Running comprehensive post-cycle cleanup...')
|
||
await this.cleanupOrphanedProcesses()
|
||
|
||
console.log('✅ Post-cycle cleanup completed - all analysis processes should be cleaned up')
|
||
}
|
||
|
||
// Signal that an analysis cycle is complete and all processes should be cleaned up
|
||
async signalAnalysisCycleComplete(): Promise<void> {
|
||
console.log('🎯 Analysis cycle completion signal received')
|
||
|
||
// Wait a bit longer to ensure all processes have had time to close
|
||
console.log('⏳ Waiting 3 seconds for graceful process shutdown...')
|
||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||
|
||
// Force cleanup of any remaining processes
|
||
console.log('🧹 Running forced cleanup after analysis cycle completion...')
|
||
await this.forceCleanupAfterCycle()
|
||
}
|
||
|
||
private async forceCleanupAfterCycle(): Promise<void> {
|
||
console.log('🚨 Force cleanup after analysis cycle - cleaning all browser processes')
|
||
|
||
try {
|
||
// Find all chromium processes
|
||
const chromiumProcesses = await this.findChromiumProcesses()
|
||
|
||
if (chromiumProcesses.length > 0) {
|
||
console.log(`🔍 Found ${chromiumProcesses.length} chromium processes to clean after cycle completion`)
|
||
|
||
// Force kill all chromium processes after cycle is complete
|
||
for (const pid of chromiumProcesses) {
|
||
try {
|
||
console.log(`🗡️ Force killing process ${pid} (cycle complete)`)
|
||
await execAsync(`kill -9 ${pid}`)
|
||
} catch (error) {
|
||
// Process might already be dead
|
||
console.log(`ℹ️ Process ${pid} may already be terminated`)
|
||
}
|
||
}
|
||
|
||
console.log(`✅ Forcefully cleaned ${chromiumProcesses.length} processes after cycle completion`)
|
||
} else {
|
||
console.log('✅ No chromium processes found - cleanup already complete')
|
||
}
|
||
|
||
// Clean up temp directories
|
||
try {
|
||
await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
|
||
await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
|
||
console.log('✅ Cleaned up temp directories and shared memory')
|
||
} catch (error) {
|
||
console.warn('⚠️ Could not clean temp directories:', error)
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('Error in force cleanup after cycle:', error)
|
||
}
|
||
}
|
||
|
||
stop(): void {
|
||
if (this.cleanupInterval) {
|
||
clearInterval(this.cleanupInterval)
|
||
this.cleanupInterval = null
|
||
}
|
||
}
|
||
}
|
||
|
||
// Initialize the aggressive cleanup
|
||
const aggressiveCleanup = AggressiveCleanup.getInstance()
|
||
|
||
export default aggressiveCleanup
|