feat: Add core search configuration for site definitions, proxies, and user agents, and introduce a browserless service.

app/services/browserless_service.py

@@ -0,0 +1,180 @@
+"""
+Browserless Service
+Manages connections to Browserless.io with advanced stealth capabilities.
+Handles:
+- Connection lifecycle
+- Stealth injection
+- Proxy rotation
+- Human behavior simulation
+"""
+
+import asyncio
+import logging
+import random
+import time
+from typing import Optional, Any
+
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+from app.core.search_config import (
+    BROWSERLESS_URL, 
+    get_amazon_proxies, 
+    get_random_user_agent,
+    COOKIE_ACCEPT_SELECTORS
+)
+
+logger = logging.getLogger(__name__)
+
+# Stealth JS to inject
+STEALTH_JS = """
+// Advanced Stealth Mode
+Object.defineProperty(navigator, 'webdriver', { get: () => false });
+delete Object.getPrototypeOf(navigator).webdriver;
+window.chrome = { runtime: {} };
+Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
+Object.defineProperty(navigator, 'languages', { get: () => ['fr-FR', 'fr', 'en-US', 'en'] });
+"""
+
+class BrowserlessService:
+    def __init__(self):
+        self._playwright = None
+        self._browser: Optional[Browser] = None
+        self._proxies = get_amazon_proxies()
+
+    async def start(self):
+        """Initialize the browser connection"""
+        if not self._playwright:
+            self._playwright = await async_playwright().start()
+        
+        if not self._browser or not self._browser.is_connected():
+            try:
+                logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+                # Use the stealth endpoint if possible, otherwise standard
+                # Note: /chromium/stealth might need specific token or config, 
+                # falling back to standard connect_over_cdp which is safer for generic browserless
+                self._browser = await self._playwright.chromium.connect_over_cdp(
+                    BROWSERLESS_URL,
+                    timeout=60000
+                )
+            except Exception as e:
+                logger.error(f"Failed to connect to Browserless: {e}")
+                raise
+
+    async def stop(self):
+        """Close resources"""
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._playwright.stop()
+
+    async def get_context(self, use_proxy: bool = False) -> BrowserContext:
+        """Create a new context with stealth settings"""
+        await self.start()
+        
+        user_agent = get_random_user_agent()
+        
+        options = {
+            "viewport": {"width": 1920, "height": 1080},
+            "user_agent": user_agent,
+            "locale": "fr-FR",
+            "timezone_id": "Europe/Paris",
+            "java_script_enabled": True,
+            "bypass_csp": True,
+            "extra_http_headers": {
+                "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Sec-Ch-Ua-Platform": '"Windows"',
+                "Upgrade-Insecure-Requests": "1",
+            }
+        }
+
+        if use_proxy and self._proxies:
+            proxy = random.choice(self._proxies)
+            options["proxy"] = proxy
+            logger.info(f"Using proxy: {proxy['server'].split('//')[1]}")
+
+        context = await self._browser.new_context(**options)
+        
+        # Inject Stealth JS
+        await context.add_init_script(STEALTH_JS)
+        
+        # Block aggressive tracking but keep images for visual verification if needed
+        # (Optimized for speed vs detection)
+        await context.route("**/*", lambda route: route.continue_())
+        
+        return context
+
+    async def simulate_human_behavior(self, page: Page):
+        """Simulate random mouse movements and scrolling"""
+        try:
+            # Mouse movements
+            for _ in range(random.randint(2, 5)):
+                await page.mouse.move(
+                    random.randint(100, 1800),
+                    random.randint(100, 900)
+                )
+                await asyncio.sleep(random.uniform(0.1, 0.3))
+            
+            # Scrolling
+            await page.evaluate("window.scrollBy(0, 300)")
+            await asyncio.sleep(random.uniform(0.5, 1.0))
+            await page.evaluate("window.scrollBy(0, -100)")
+        except Exception:
+            pass
+
+    async def handle_cookie_banners(self, page: Page):
+        """Attempt to close cookie banners"""
+        try:
+            for selector in COOKIE_ACCEPT_SELECTORS:
+                if await page.locator(selector).first.is_visible(timeout=200):
+                    await page.locator(selector).first.click()
+                    logger.info(f"Closed cookie banner with {selector}")
+                    await asyncio.sleep(0.5)
+                    return
+        except Exception:
+            pass
+
+    async def get_page_content(self, url: str, use_proxy: bool = False, wait_selector: str = None) -> tuple[str, str]:
+        """
+        Fetch page content with full stealth lifecycle.
+        Returns (html_content, screenshot_path)
+        """
+        context = await self.get_context(use_proxy=use_proxy)
+        page = await context.new_page()
+        
+        try:
+            logger.info(f"Navigating to {url}")
+            # Random delay before start
+            await asyncio.sleep(random.uniform(0.5, 1.5))
+            
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=60000)
+            
+            if response.status == 503:
+                logger.warning(f"503 Detected on {url}")
+                # Retry logic could be handled here or by caller, 
+                # for now we return what we have, caller decides
+            
+            await self.handle_cookie_banners(page)
+            await self.simulate_human_behavior(page)
+            
+            if wait_selector:
+                try:
+                    await page.wait_for_selector(wait_selector, timeout=10000)
+                except Exception:
+                    logger.warning(f"Wait selector {wait_selector} timed out")
+
+            # Wait for network idle to ensure dynamic content loads
+            try:
+                await page.wait_for_load_state("networkidle", timeout=5000)
+            except Exception:
+                pass
+
+            content = await page.content()
+            return content, "" # Screenshot logic can be added if needed
+
+        except Exception as e:
+            logger.error(f"Error fetching {url}: {e}")
+            return "", ""
+        finally:
+            await context.close()
+
+# Global instance
+browserless_service = BrowserlessService()