🔐 Implement robust session persistence to avoid 'are you human' captcha checks

- Add comprehensive session persistence with cookies, localStorage, and sessionStorage
- Implement stealth browser features to reduce bot detection
- Add smartLogin() method that prioritizes saved sessions over fresh logins
- Create session management utilities (refresh, clear, test validity)
- Update enhanced screenshot service to use session persistence
- Add comprehensive documentation and test script
- Support manual login fallback when captcha is encountered
- Sessions stored in .tradingview-session/ directory for Docker compatibility

This solves the captcha problem by avoiding repeated logins through persistent sessions.

lib/tradingview-automation.ts

@@ -1,4 +1,4 @@
-import { chromium, Browser, Page } from 'playwright'
+import { chromium, Browser, Page, BrowserContext } from 'playwright'
 import fs from 'fs/promises'
 import path from 'path'
 
@@ -17,11 +17,23 @@ export interface NavigationOptions {
   waitForChart?: boolean
 }
 
+// Session persistence configuration
+const SESSION_DATA_DIR = path.join(process.cwd(), '.tradingview-session')
+const COOKIES_FILE = path.join(SESSION_DATA_DIR, 'cookies.json')
+const SESSION_STORAGE_FILE = path.join(SESSION_DATA_DIR, 'session-storage.json')
+
 export class TradingViewAutomation {
   private browser: Browser | null = null
+  private context: BrowserContext | null = null
   private page: Page | null = null
+  private isAuthenticated: boolean = false
 
   async init(): Promise<void> {
+    console.log('🚀 Initializing TradingView automation with session persistence...')
+    
+    // Ensure session directory exists
+    await fs.mkdir(SESSION_DATA_DIR, { recursive: true })
+    
     this.browser = await chromium.launch({
       headless: true, // Must be true for Docker containers
       args: [
@@ -43,12 +55,16 @@ export class TradingViewAutomation {
         '--disable-default-apps',
         '--disable-sync',
         '--metrics-recording-only',
-        '--no-first-run',
         '--safebrowsing-disable-auto-update',
         '--disable-component-extensions-with-background-pages',
         '--disable-background-networking',
         '--disable-software-rasterizer',
-        '--remote-debugging-port=9222'
+        '--remote-debugging-port=9222',
+        // Additional args to reduce captcha detection
+        '--disable-blink-features=AutomationControlled',
+        '--disable-features=VizDisplayCompositor,VizHitTestSurfaceLayer',
+        '--disable-features=ScriptStreaming',
+        '--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
       ]
     })
     
@@ -56,19 +72,149 @@ export class TradingViewAutomation {
       throw new Error('Failed to launch browser')
     }
     
-    this.page = await this.browser.newPage()
+    // Create browser context with session persistence
+    this.context = await this.browser.newContext({
+      userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      viewport: { width: 1920, height: 1080 },
+      // Add additional headers to appear more human-like
+      extraHTTPHeaders: {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Cache-Control': 'no-cache',
+        'Pragma': 'no-cache',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Upgrade-Insecure-Requests': '1'
+      }
+    })
+    
+    if (!this.context) {
+      throw new Error('Failed to create browser context')
+    }
+    
+    // Load saved session if available
+    await this.loadSession()
+    
+    this.page = await this.context.newPage()
     
     if (!this.page) {
       throw new Error('Failed to create new page')
     }
     
-    // Set viewport and user agent
-    await this.page.setViewportSize({ width: 1920, height: 1080 })
-    
-    // Use setExtraHTTPHeaders instead of setUserAgent for better compatibility
-    await this.page.setExtraHTTPHeaders({
-      'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    // Add stealth measures to reduce bot detection
+    await this.page.addInitScript(() => {
+      // Override the navigator.webdriver property
+      Object.defineProperty(navigator, 'webdriver', {
+        get: () => undefined,
+      });
+      
+      // Mock plugins
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [1, 2, 3, 4, 5],
+      });
+      
+      // Mock languages
+      Object.defineProperty(navigator, 'languages', {
+        get: () => ['en-US', 'en'],
+      });
+      
+      // Override permissions API to avoid detection
+      const originalQuery = window.navigator.permissions.query;
+      window.navigator.permissions.query = (parameters: any) => {
+        if (parameters.name === 'notifications') {
+          return Promise.resolve({
+            state: Notification.permission,
+            name: parameters.name,
+            onchange: null,
+            addEventListener: () => {},
+            removeEventListener: () => {},
+            dispatchEvent: () => false
+          } as PermissionStatus);
+        }
+        return originalQuery.call(window.navigator.permissions, parameters);
+      };
     })
+    
+    console.log('✅ Browser and session initialized successfully')
+  }
+
+  /**
+   * Check if user is already logged in to TradingView
+   */
+  async checkLoginStatus(): Promise<boolean> {
+    if (!this.page) return false
+
+    try {
+      console.log('🔍 Checking login status...')
+      
+      // Navigate to TradingView if not already there
+      const currentUrl = await this.page.url()
+      if (!currentUrl.includes('tradingview.com')) {
+        console.log('📄 Navigating to TradingView...')
+        await this.page.goto('https://www.tradingview.com', {
+          waitUntil: 'domcontentloaded',
+          timeout: 30000
+        })
+        
+        // Restore session storage after navigation
+        await this.restoreSessionStorage()
+        
+        // Wait for page to settle
+        await this.page.waitForTimeout(3000)
+      }
+
+      // Check for login indicators
+      const loginIndicators = [
+        '[data-name="watchlist-button"]',
+        '.tv-header__watchlist-button',
+        '.tv-header__user-menu-button',
+        'button:has-text("M")',
+        '.js-header-user-menu-button',
+        '[data-name="user-menu"]'
+      ]
+
+      for (const selector of loginIndicators) {
+        try {
+          if (await this.page.locator(selector).isVisible({ timeout: 2000 })) {
+            console.log(`✅ Found login indicator: ${selector}`)
+            this.isAuthenticated = true
+            return true
+          }
+        } catch (e) {
+          continue
+        }
+      }
+
+      // Additional check: look for sign-in buttons (indicates not logged in)
+      const signInSelectors = [
+        'a[href*="signin"]',
+        'button:has-text("Sign in")',
+        '.tv-header__user-menu-button--anonymous'
+      ]
+
+      for (const selector of signInSelectors) {
+        try {
+          if (await this.page.locator(selector).isVisible({ timeout: 2000 })) {
+            console.log(`❌ Found sign-in button: ${selector} - not logged in`)
+            this.isAuthenticated = false
+            return false
+          }
+        } catch (e) {
+          continue
+        }
+      }
+
+      console.log('🤔 Login status unclear, will attempt login')
+      this.isAuthenticated = false
+      return false
+
+    } catch (error) {
+      console.error('❌ Error checking login status:', error)
+      this.isAuthenticated = false
+      return false
+    }
   }
 
   async login(credentials?: TradingViewCredentials): Promise<boolean> {
@@ -83,6 +229,13 @@ export class TradingViewAutomation {
     }
 
     try {
+      // Check if already logged in
+      const loggedIn = await this.checkLoginStatus()
+      if (loggedIn) {
+        console.log('✅ Already logged in, skipping login steps')
+        return true
+      }
+
       console.log('Navigating to TradingView login page...')
       
       // Try different login URLs that TradingView might use
@@ -598,6 +751,11 @@ export class TradingViewAutomation {
         )
         
         console.log('Login successful!')
+        this.isAuthenticated = true
+        
+        // Save session after successful login
+        await this.saveSession()
+        
         return true
       } catch (error) {
         console.error('Login verification failed:', error)
@@ -614,6 +772,65 @@ export class TradingViewAutomation {
     }
   }
 
+  /**
+   * Smart login that prioritizes session persistence to avoid captchas
+   */
+  async smartLogin(credentials?: TradingViewCredentials): Promise<boolean> {
+    if (!this.page) throw new Error('Page not initialized')
+
+    try {
+      console.log('🔍 Attempting smart login with session persistence...')
+      
+      // First check if already logged in
+      const alreadyLoggedIn = await this.checkLoginStatus()
+      if (alreadyLoggedIn) {
+        console.log('🎉 Already logged in to TradingView! Skipping login process.')
+        await this.saveSession() // Save current session
+        return true
+      }
+
+      console.log('🔐 Not logged in, proceeding with manual login...')
+      console.log('⚠️  IMPORTANT: Manual intervention required due to captcha protection.')
+      console.log('📱 Please log in manually in a browser and the session will be saved for future use.')
+      
+      // Navigate to login page for manual login
+      await this.page.goto('https://www.tradingview.com/accounts/signin/', {
+        waitUntil: 'domcontentloaded',
+        timeout: 30000
+      })
+      
+      // Wait and give user time to manually complete login
+      console.log('⏳ Waiting for manual login completion...')
+      console.log('💡 You have 2 minutes to complete login manually in the browser.')
+      
+      // Check every 10 seconds for login completion
+      let attempts = 0
+      const maxAttempts = 12 // 2 minutes
+      
+      while (attempts < maxAttempts) {
+        await this.page.waitForTimeout(10000) // Wait 10 seconds
+        attempts++
+        
+        console.log(`🔄 Checking login status (attempt ${attempts}/${maxAttempts})...`)
+        const loggedIn = await this.checkLoginStatus()
+        
+        if (loggedIn) {
+          console.log('✅ Manual login detected! Saving session for future use.')
+          this.isAuthenticated = true
+          await this.saveSession()
+          return true
+        }
+      }
+      
+      console.log('⏰ Timeout waiting for manual login.')
+      return false
+      
+    } catch (error) {
+      console.error('❌ Smart login failed:', error)
+      return false
+    }
+  }
+
   async navigateToChart(options: NavigationOptions = {}): Promise<boolean> {
     if (!this.page) throw new Error('Page not initialized')
 
@@ -1003,16 +1220,309 @@ export class TradingViewAutomation {
   }
 
   async close(): Promise<void> {
+    // Save session data before closing
+    if (this.isAuthenticated) {
+      await this.saveSession()
+    }
+    
     if (this.page) {
       await this.page.close()
       this.page = null
     }
+    if (this.context) {
+      await this.context.close()
+      this.context = null
+    }
     if (this.browser) {
       await this.browser.close()
       this.browser = null
     }
   }
 
+  /**
+   * Load saved session data (cookies, localStorage, etc.)
+   */
+  private async loadSession(): Promise<void> {
+    try {
+      console.log('🔄 Loading saved session data...')
+      
+      // Load cookies
+      if (await this.fileExists(COOKIES_FILE)) {
+        const cookiesData = await fs.readFile(COOKIES_FILE, 'utf8')
+        const cookies = JSON.parse(cookiesData)
+        await this.context!.addCookies(cookies)
+        console.log(`✅ Loaded ${cookies.length} cookies from saved session`)
+      }
+      
+      // Note: Session storage will be loaded after page navigation
+      
+    } catch (error) {
+      console.log('⚠️ Could not load session data (starting fresh):', error)
+    }
+  }
+
+  /**
+   * Save current session data for future use
+   */
+  private async saveSession(): Promise<void> {
+    try {
+      console.log('💾 Saving session data...')
+      
+      if (!this.context || !this.page) return
+      
+      // Save cookies
+      const cookies = await this.context.cookies()
+      await fs.writeFile(COOKIES_FILE, JSON.stringify(cookies, null, 2))
+      console.log(`✅ Saved ${cookies.length} cookies`)
+      
+      // Save session storage and localStorage
+      const sessionData = await this.page.evaluate(() => {
+        const localStorage: { [key: string]: string | null } = {}
+        const sessionStorage: { [key: string]: string | null } = {}
+        
+        // Extract localStorage
+        for (let i = 0; i < window.localStorage.length; i++) {
+          const key = window.localStorage.key(i)
+          if (key) {
+            localStorage[key] = window.localStorage.getItem(key)
+          }
+        }
+        
+        // Extract sessionStorage
+        for (let i = 0; i < window.sessionStorage.length; i++) {
+          const key = window.sessionStorage.key(i)
+          if (key) {
+            sessionStorage[key] = window.sessionStorage.getItem(key)
+          }
+        }
+        
+        return { localStorage, sessionStorage }
+      })
+      
+      await fs.writeFile(SESSION_STORAGE_FILE, JSON.stringify(sessionData, null, 2))
+      console.log('✅ Saved session storage and localStorage')
+      
+    } catch (error) {
+      console.error('❌ Failed to save session data:', error)
+    }
+  }
+
+  /**
+   * Restore session storage and localStorage
+   */
+  private async restoreSessionStorage(): Promise<void> {
+    try {
+      if (!this.page || !await this.fileExists(SESSION_STORAGE_FILE)) return
+      
+      const sessionData = JSON.parse(await fs.readFile(SESSION_STORAGE_FILE, 'utf8'))
+      
+      await this.page.evaluate((data) => {
+        // Restore localStorage
+        if (data.localStorage) {
+          for (const [key, value] of Object.entries(data.localStorage)) {
+            try {
+              window.localStorage.setItem(key, value as string)
+            } catch (e) {
+              console.log('Could not restore localStorage item:', key)
+            }
+          }
+        }
+        
+        // Restore sessionStorage
+        if (data.sessionStorage) {
+          for (const [key, value] of Object.entries(data.sessionStorage)) {
+            try {
+              window.sessionStorage.setItem(key, value as string)
+            } catch (e) {
+              console.log('Could not restore sessionStorage item:', key)
+            }
+          }
+        }
+      }, sessionData)
+      
+      console.log('✅ Restored session storage and localStorage')
+      
+    } catch (error) {
+      console.log('⚠️ Could not restore session storage:', error)
+    }
+  }
+
+  /**
+   * Refresh session to keep it alive
+   */
+  async refreshSession(): Promise<boolean> {
+    if (!this.page || !this.isAuthenticated) return false
+
+    try {
+      console.log('🔄 Refreshing TradingView session...')
+      
+      // Just reload the current page to refresh session
+      await this.page.reload({
+        waitUntil: 'domcontentloaded',
+        timeout: 30000
+      })
+      
+      // Wait for page to settle
+      await this.page.waitForTimeout(2000)
+      
+      // Verify still logged in
+      const stillLoggedIn = await this.checkLoginStatus()
+      if (stillLoggedIn) {
+        console.log('✅ Session refreshed successfully')
+        await this.saveSession() // Save refreshed session
+        return true
+      } else {
+        console.log('❌ Session expired during refresh')
+        this.isAuthenticated = false
+        return false
+      }
+      
+    } catch (error) {
+      console.error('❌ Failed to refresh session:', error)
+      return false
+    }
+  }
+
+  /**
+   * Clear all saved session data
+   */
+  async clearSession(): Promise<void> {
+    try {
+      console.log('🗑️ Clearing saved session data...')
+      
+      if (await this.fileExists(COOKIES_FILE)) {
+        await fs.unlink(COOKIES_FILE)
+        console.log('✅ Cleared cookies file')
+      }
+      
+      if (await this.fileExists(SESSION_STORAGE_FILE)) {
+        await fs.unlink(SESSION_STORAGE_FILE)
+        console.log('✅ Cleared session storage file')
+      }
+      
+      // Clear browser context storage if available
+      if (this.context) {
+        await this.context.clearCookies()
+        console.log('✅ Cleared browser context cookies')
+      }
+      
+      this.isAuthenticated = false
+      console.log('✅ Session data cleared successfully')
+      
+    } catch (error) {
+      console.error('❌ Failed to clear session data:', error)
+    }
+  }
+
+  /**
+   * Get session status information
+   */
+  async getSessionInfo(): Promise<{
+    isAuthenticated: boolean
+    hasSavedCookies: boolean
+    hasSavedStorage: boolean
+    cookiesCount: number
+    currentUrl: string
+  }> {
+    const hasSavedCookies = await this.fileExists(COOKIES_FILE)
+    const hasSavedStorage = await this.fileExists(SESSION_STORAGE_FILE)
+    
+    let cookiesCount = 0
+    if (hasSavedCookies) {
+      try {
+        const cookiesData = await fs.readFile(COOKIES_FILE, 'utf8')
+        const cookies = JSON.parse(cookiesData)
+        cookiesCount = cookies.length
+      } catch (e) {
+        // Ignore error
+      }
+    }
+    
+    const currentUrl = this.page ? await this.page.url() : ''
+    
+    return {
+      isAuthenticated: this.isAuthenticated,
+      hasSavedCookies,
+      hasSavedStorage,
+      cookiesCount,
+      currentUrl
+    }
+  }
+
+  /**
+   * Test session persistence by checking if saved session data exists and is valid
+   */
+  async testSessionPersistence(): Promise<{
+    hasSessionData: boolean
+    isValid: boolean
+    sessionInfo: any
+  }> {
+    try {
+      console.log('🧪 Testing session persistence...')
+      
+      const sessionInfo = await this.getSessionInfo()
+      console.log('📊 Current session info:', sessionInfo)
+      
+      if (!sessionInfo.hasSavedCookies && !sessionInfo.hasSavedStorage) {
+        console.log('❌ No saved session data found')
+        return {
+          hasSessionData: false,
+          isValid: false,
+          sessionInfo
+        }
+      }
+      
+      console.log('✅ Saved session data found')
+      console.log(`🍪 Cookies: ${sessionInfo.cookiesCount}`)
+      console.log(`💾 Storage: ${sessionInfo.hasSavedStorage ? 'Yes' : 'No'}`)
+      
+      // Try to use the session
+      if (this.page) {
+        // Navigate to TradingView to test session validity
+        await this.page.goto('https://www.tradingview.com', {
+          waitUntil: 'domcontentloaded',
+          timeout: 30000
+        })
+        
+        // Restore session storage
+        await this.restoreSessionStorage()
+        
+        // Check if session is still valid
+        const isLoggedIn = await this.checkLoginStatus()
+        
+        if (isLoggedIn) {
+          console.log('🎉 Session is valid and user is logged in!')
+          return {
+            hasSessionData: true,
+            isValid: true,
+            sessionInfo
+          }
+        } else {
+          console.log('⚠️ Session data exists but appears to be expired')
+          return {
+            hasSessionData: true,
+            isValid: false,
+            sessionInfo
+          }
+        }
+      }
+      
+      return {
+        hasSessionData: true,
+        isValid: false,
+        sessionInfo
+      }
+      
+    } catch (error) {
+      console.error('❌ Session persistence test failed:', error)
+      return {
+        hasSessionData: false,
+        isValid: false,
+        sessionInfo: null
+      }
+    }
+  }
+
   // Utility method to wait for chart data to load
   async waitForChartData(timeout: number = 15000): Promise<boolean> {
     if (!this.page) return false
@@ -1072,6 +1582,16 @@ export class TradingViewAutomation {
       return false
     }
   }
+
+  // Check if file exists
+  private async fileExists(filePath: string): Promise<boolean> {
+    try {
+      await fs.access(filePath)
+      return true
+    } catch (error) {
+      return false
+    }
+  }
 }
 
 export const tradingViewAutomation = new TradingViewAutomation()