trading_bot_v3/lib/aggressive-cleanup.ts

// Aggressive process cleanup utility
import { exec } from 'child_process'
import { promisify } from 'util'

const execAsync = promisify(exec)

class AggressiveCleanup {
  private static instance: AggressiveCleanup
  private cleanupInterval: NodeJS.Timeout | null = null
  private isRunning = false
  private isInitialized = false

  private constructor() {
    // Don't auto-start - let startup.ts control it
  }

  static getInstance(): AggressiveCleanup {
    if (!AggressiveCleanup.instance) {
      AggressiveCleanup.instance = new AggressiveCleanup()
    }
    return AggressiveCleanup.instance
  }

  startPeriodicCleanup() {
    if (this.isInitialized) {
      console.log('🔄 Aggressive cleanup already initialized')
      return
    }

    this.isInitialized = true
    console.log('🚀 Starting aggressive cleanup system')

    // In development, use on-demand cleanup instead of periodic
    if (process.env.NODE_ENV === 'development') {
      console.log('🔧 Development mode: Using on-demand cleanup (triggered after analysis)')
      console.log('✅ On-demand cleanup system ready')
      return
    }

    // Production: Clean up every 10 minutes (longer intervals)
    this.cleanupInterval = setInterval(async () => {
      try {
        await this.cleanupOrphanedProcesses()
      } catch (error) {
        console.error('Error in periodic cleanup:', error)
      }
    }, 10 * 60 * 1000) // 10 minutes

    // Also run initial cleanup after 60 seconds
    setTimeout(() => {
      this.cleanupOrphanedProcesses().catch(console.error)
    }, 60000)

    console.log('✅ Periodic cleanup system started (10 min intervals)')
  }

  async cleanupOrphanedProcesses(): Promise<void> {
    if (this.isRunning) {
      console.log('🔒 Cleanup already in progress, skipping...')
      return
    }
    
    // Check if auto cleanup is disabled (for development)
    if (process.env.DISABLE_AUTO_CLEANUP === 'true') {
      console.log('🚫 Auto cleanup disabled via DISABLE_AUTO_CLEANUP environment variable')
      return
    }
    
    this.isRunning = true
    const isDevelopment = process.env.NODE_ENV === 'development'
    const cleanupType = isDevelopment ? 'gentle' : 'aggressive'
    
    console.log(`🧹 Running ${cleanupType} cleanup for orphaned processes...`)

    try {
      // Check for active analysis sessions
      try {
        const { progressTracker } = await import('./progress-tracker')
        const activeSessions = progressTracker.getActiveSessions()
        
        if (activeSessions.length > 0) {
          console.log(`⚠️  Skipping cleanup - ${activeSessions.length} active analysis sessions detected:`)
          activeSessions.forEach(session => {
            const progress = progressTracker.getProgress(session)
            if (progress) {
              const activeStep = progress.steps.find(step => step.status === 'active')
              const currentStep = activeStep ? activeStep.title : 'Unknown'
              console.log(`  - ${session}: ${currentStep} (Step ${progress.currentStep}/${progress.totalSteps})`)
            } else {
              console.log(`  - ${session}: Session info not available`)
            }
          })
          console.log('ℹ️  Will retry cleanup after analysis completes')
          return
        }
        
        console.log('✅ No active analysis sessions detected, proceeding with cleanup')
      } catch (importError) {
        console.warn('⚠️  Could not check active sessions, proceeding cautiously with cleanup')
        console.warn('Import error:', importError)
        
        // In case of import errors, be extra cautious - only clean very old processes
        if (isDevelopment) {
          console.log('🔧 Development mode with import issues - using aggressive cleanup to clear stuck processes')
          // In development, if we can't check sessions, assume they're stuck and clean aggressively
        }
      }

      // Find and kill orphaned chromium processes
      const chromiumProcesses = await this.findChromiumProcesses()
      
      if (chromiumProcesses.length > 0) {
        console.log(`🔍 Found ${chromiumProcesses.length} chromium processes, evaluating for cleanup...`)
        
        // In development, be more selective about which processes to kill
        let processesToKill = chromiumProcesses
        
        if (isDevelopment) {
          // Only kill processes that are likely orphaned (older than 5 minutes)
          const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 5 * 60 * 1000) // 5 minutes
          processesToKill = oldProcesses
          
          if (processesToKill.length === 0) {
            console.log('✅ All chromium processes appear to be recent and potentially active - skipping cleanup')
            return
          }
          
          console.log(`🔧 Development mode: Cleaning only ${processesToKill.length} old processes (older than 5 minutes)`)
        }
        
        for (const pid of processesToKill) {
          try {
            if (isDevelopment) {
              // In development, use gentler SIGTERM first
              console.log(`🔧 Dev mode: Gentle shutdown of process ${pid}`)
              await execAsync(`kill -TERM ${pid}`)
              // Give process 3 seconds to shut down gracefully
              await new Promise(resolve => setTimeout(resolve, 3000))
              
              // Check if process is still running
              try {
                await execAsync(`kill -0 ${pid}`)
                // Process still running, force kill
                console.log(`⚠️  Process ${pid} didn't shut down gracefully, force killing`)
                await execAsync(`kill -9 ${pid}`)
              } catch {
                // Process already dead, that's good
                console.log(`✅ Process ${pid} shut down gracefully`)
              }
            } else {
              // Production: immediate force kill
              await execAsync(`kill -9 ${pid}`)
              console.log(`✅ Killed process ${pid}`)
            }
          } catch (error) {
            // Process might already be dead
            console.log(`ℹ️  Process ${pid} may already be terminated`)
          }
        }
      } else {
        console.log('✅ No orphaned chromium processes found')
      }

      // Clean up temp directories
      try {
        await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
        console.log('✅ Cleaned up temp directories')
      } catch (error) {
        // Ignore errors
      }

      // Clean up shared memory
      try {
        await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
        console.log('✅ Cleaned up shared memory')
      } catch (error) {
        // Ignore errors
      }

    } catch (error) {
      console.error(`Error in ${cleanupType} cleanup:`, error)
    } finally {
      this.isRunning = false
      console.log(`🏁 ${cleanupType} cleanup completed`)
    }
  }

  private async findChromiumProcesses(): Promise<string[]> {
    try {
      const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep | awk \'{print $2}\'')
      return stdout.trim().split('\n').filter((pid: string) => pid && pid !== '')
    } catch (error) {
      return []
    }
  }

  private async filterOldProcesses(pids: string[], maxAgeMs: number): Promise<string[]> {
    const oldProcesses: string[] = []
    
    for (const pid of pids) {
      try {
        // Get process start time
        const { stdout } = await execAsync(`ps -o pid,lstart -p ${pid} | tail -1`)
        const processInfo = stdout.trim()
        
        if (processInfo) {
          // Parse the process start time
          const parts = processInfo.split(/\s+/)
          if (parts.length >= 6) {
            // Format: PID Mon DD HH:MM:SS YYYY
            const startTimeStr = parts.slice(1).join(' ')
            const startTime = new Date(startTimeStr)
            const now = new Date()
            const processAge = now.getTime() - startTime.getTime()
            
            if (processAge > maxAgeMs) {
              console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - marked for cleanup`)
              oldProcesses.push(pid)
            } else {
              console.log(`🕐 Process ${pid} is ${Math.round(processAge / 60000)} minutes old - keeping alive`)
            }
          }
        }
      } catch (error) {
        // If we can't get process info, assume it's old and safe to clean
        console.log(`❓ Could not get age info for process ${pid} - assuming it's old`)
        oldProcesses.push(pid)
      }
    }
    
    return oldProcesses
  }

  async forceCleanup(): Promise<void> {
    console.log('🚨 Force cleanup initiated...')
    
    // Stop periodic cleanup
    if (this.cleanupInterval) {
      clearInterval(this.cleanupInterval)
    }

    // Run aggressive cleanup
    await this.cleanupOrphanedProcesses()
    
    // Kill all chromium processes
    try {
      await execAsync('pkill -9 -f "chromium" 2>/dev/null || true')
      await execAsync('pkill -9 -f "chrome" 2>/dev/null || true')
      console.log('✅ Force killed all browser processes')
    } catch (error) {
      console.error('Error in force cleanup:', error)
    }
  }

  // New method for on-demand cleanup after complete automation cycle
  async runPostAnalysisCleanup(): Promise<void> {
    // Check if auto cleanup is disabled (for development)
    if (process.env.DISABLE_AUTO_CLEANUP === 'true') {
      console.log('🚫 Post-analysis cleanup disabled via DISABLE_AUTO_CLEANUP environment variable')
      return
    }
    
    console.log('🧹 Post-cycle cleanup triggered (analysis + decision complete)...')
    
    // Wait for all browser processes to fully close
    console.log('⏳ Waiting 3 seconds for all processes to close gracefully...')
    await new Promise(resolve => setTimeout(resolve, 3000))
    
    // Always run cleanup after complete automation cycle - don't check for active sessions
    // since the analysis is complete and we need to ensure all processes are cleaned up
    console.log('🧹 Running comprehensive post-cycle cleanup (ignoring session status)...')
    
    try {
      // Find all chromium processes
      const chromiumProcesses = await this.findChromiumProcesses()
      
      if (chromiumProcesses.length === 0) {
        console.log('✅ No chromium processes found to clean up')
        return
      }
      
      console.log(`🔍 Found ${chromiumProcesses.length} chromium processes for post-analysis cleanup`)
      
      // In post-analysis cleanup, we're more aggressive since analysis is complete
      // Try graceful shutdown first
      for (const pid of chromiumProcesses) {
        try {
          console.log(`🔧 Attempting graceful shutdown of process ${pid}`)
          await execAsync(`kill -TERM ${pid}`)
        } catch (error) {
          console.log(`ℹ️  Process ${pid} may already be terminated`)
        }
      }
      
      // Wait for graceful shutdown
      await new Promise(resolve => setTimeout(resolve, 5000))
      
      // Check which processes are still running and force kill them
      const stillRunning = await this.findStillRunningProcesses(chromiumProcesses)
      
      if (stillRunning.length > 0) {
        console.log(`🗡️  Force killing ${stillRunning.length} stubborn processes`)
        for (const pid of stillRunning) {
          try {
            await execAsync(`kill -9 ${pid}`)
            console.log(`💀 Force killed process ${pid}`)
          } catch (error) {
            console.log(`ℹ️  Process ${pid} already terminated`)
          }
        }
      } else {
        console.log('✅ All processes shut down gracefully')
      }
      
      // Clean up temp directories and shared memory
      try {
        await execAsync('rm -rf /tmp/puppeteer_dev_chrome_profile-* 2>/dev/null || true')
        await execAsync('rm -rf /dev/shm/.org.chromium.* 2>/dev/null || true')
        await execAsync('rm -rf /tmp/.org.chromium.* 2>/dev/null || true')
        console.log('✅ Cleaned up temporary files and shared memory')
      } catch (error) {
        console.error('Warning: Could not clean up temporary files:', error)
      }
      
      console.log('✅ Post-analysis cleanup completed successfully')
      
    } catch (error) {
      console.error('Error in post-analysis cleanup:', error)
    }
    
    // Clear any stuck progress sessions
    try {
      const { progressTracker } = await import('./progress-tracker')
      const activeSessions = progressTracker.getActiveSessions()
      
      if (activeSessions.length > 0) {
        console.log(`🧹 Force clearing ${activeSessions.length} potentially stuck sessions`)
        activeSessions.forEach(session => {
          console.log(`🧹 Force clearing session: ${session}`)
          progressTracker.deleteSession(session)
        })
      }
    } catch (error) {
      console.warn('Could not clear progress sessions:', error)
    }
  }

  // Signal that an analysis cycle is complete and all processes should be cleaned up
  async signalAnalysisCycleComplete(): Promise<void> {
    console.log('🎯 Analysis cycle completion signal received')
    
    // Wait for graceful shutdown of analysis-related processes
    console.log('⏳ Waiting 5 seconds for graceful process shutdown...')
    await new Promise(resolve => setTimeout(resolve, 5000))
    
    // Check if there are any active progress sessions first
    const activeSessions = await this.checkActiveAnalysisSessions()
    if (activeSessions > 0) {
      console.log(`⚠️  Found ${activeSessions} active analysis sessions, skipping aggressive cleanup`)
      return
    }
    
    // Only run cleanup if no active sessions
    console.log('🧹 No active sessions detected, running post-analysis cleanup...')
    await this.cleanupPostAnalysisProcesses()
  }

  private async checkActiveAnalysisSessions(): Promise<number> {
    // Check if progress tracker has any active sessions
    try {
      // This is a simple check - in a real scenario you might want to check actual session state
      const { stdout } = await execAsync('pgrep -f "automation-.*-.*" | wc -l')
      return parseInt(stdout.trim()) || 0
    } catch (error) {
      return 0
    }
  }

  private async cleanupPostAnalysisProcesses(): Promise<void> {
    console.log('🚨 Post-analysis cleanup - targeting orphaned browser processes')
    
    try {
      // Find all chromium processes
      const chromiumProcesses = await this.findChromiumProcesses()
      
      if (chromiumProcesses.length === 0) {
        console.log('✅ No chromium processes found to clean up')
        return
      }
      
      console.log(`🔍 Found ${chromiumProcesses.length} chromium processes`)
      
      // Filter out processes that are too new (less than 2 minutes old)
      const oldProcesses = await this.filterOldProcesses(chromiumProcesses, 2 * 60) // 2 minutes
      
      if (oldProcesses.length === 0) {
        console.log('✅ All chromium processes are recent, not cleaning up')
        return
      }
      
      console.log(`🧹 Cleaning up ${oldProcesses.length} old chromium processes`)
      
      // Try graceful shutdown first
      for (const pid of oldProcesses) {
        try {
          console.log(`<EFBFBD> Attempting graceful shutdown of process ${pid}`)
          await execAsync(`kill -TERM ${pid}`)
        } catch (error) {
          console.log(`ℹ️  Process ${pid} may already be terminated`)
        }
      }
      
      // Wait for graceful shutdown
      await new Promise(resolve => setTimeout(resolve, 3000))
      
      // Check which processes are still running and force kill only those
      const stillRunning = await this.findStillRunningProcesses(oldProcesses)
      
      if (stillRunning.length > 0) {
        console.log(`🗡️  Force killing ${stillRunning.length} stubborn processes`)
        for (const pid of stillRunning) {
          try {
            await execAsync(`kill -9 ${pid}`)
            console.log(`💀 Force killed process ${pid}`)
          } catch (error) {
            console.log(`ℹ️  Process ${pid} already terminated`)
          }
        }
      }
      
      console.log('✅ Post-analysis cleanup completed')
      
    } catch (error) {
      console.error('Error in post-analysis cleanup:', error)
    }
  }

  private async findStillRunningProcesses(pids: string[]): Promise<string[]> {
    const stillRunning: string[] = []
    
    for (const pid of pids) {
      try {
        await execAsync(`kill -0 ${pid}`) // Check if process exists
        stillRunning.push(pid)
      } catch (error) {
        // Process is already dead
      }
    }
    
    return stillRunning
  }

  // Method to get detailed process information for debugging
  async getProcessInfo(): Promise<void> {
    try {
      console.log('🔍 Current browser process information:')
      
      // Get all chromium processes with detailed info
      const { stdout } = await execAsync('ps aux | grep -E "(chromium|chrome)" | grep -v grep')
      const processes = stdout.trim().split('\n').filter(line => line.length > 0)
      
      if (processes.length === 0) {
        console.log('✅ No browser processes currently running')
        return
      }
      
      console.log(`📊 Found ${processes.length} browser processes:`)
      processes.forEach((process, index) => {
        const parts = process.split(/\s+/)
        const pid = parts[1]
        const cpu = parts[2]
        const mem = parts[3]
        const command = parts.slice(10).join(' ')
        console.log(`  ${index + 1}. PID: ${pid}, CPU: ${cpu}%, MEM: ${mem}%, CMD: ${command.substring(0, 100)}...`)
      })
      
      // Get memory usage
      const { stdout: memInfo } = await execAsync('free -h')
      console.log('💾 Memory usage:')
      console.log(memInfo)
      
    } catch (error) {
      console.error('Error getting process info:', error)
    }
  }

  stop(): void {
    if (this.cleanupInterval) {
      clearInterval(this.cleanupInterval)
      this.cleanupInterval = null
    }
  }
}

// Initialize the aggressive cleanup
const aggressiveCleanup = AggressiveCleanup.getInstance()

export default aggressiveCleanup