feat: Implement BrowserlessService for robust browser automation with auto-reconnection, popup handling, and price extraction capabilities.

app/services/browserless_service.py

@@ -5,11 +5,14 @@ Manages connections to Browserless.io with advanced robustness features:
 - Advanced popup handling
 - Flexible configuration
 - Thread-safe operations
+- Amazon & Generic price extraction
 """
 
 import asyncio
 import logging
 import os
+import re
+import time
 from dataclasses import dataclass
 from datetime import datetime
 
@@ -37,7 +40,7 @@ POPUP_SELECTORS = [
     "button:has-text('No thanks')",
     "a:has-text('No, thanks')",
     "div[role='dialog'] button[aria-label='Close']",
-    # Cookie banners (kept for compatibility)
+    # Cookie banners
     "#sp-cc-accept",
     "#onetrust-accept-btn-handler",
     "button:has-text('Tout accepter')",
@@ -100,7 +103,7 @@ class BrowserlessService:
                     await cls._initialize()
                     return cls._browser is not None
 
-                # Test if browser is still alive by trying to create a context
+                # Test if browser is still alive
                 try:
                     test_context = await cls._browser.new_context()
                     await test_context.close()
@@ -108,7 +111,6 @@ class BrowserlessService:
                 except Exception as e:
                     logger.error(f"Browser connection test failed: {e}")
                     logger.info("Attempting to reconnect browser...")
-                    # Clear the old browser
                     cls._browser = None
                     if cls._playwright:
                         try:
@@ -116,7 +118,6 @@ class BrowserlessService:
                         except Exception:
                             pass
                         cls._playwright = None
-                    # Reconnect
                     await cls._initialize()
                     return cls._browser is not None
             except Exception as e:
@@ -148,13 +149,10 @@ class BrowserlessService:
             },
         }
 
-        # Proxy support (can be enhanced with proxy pool in future)
         if use_proxy:
             logger.info("Proxy support requested (not implemented in this version)")
 
         context = await browser.new_context(**options)
-
-        # Block aggressive tracking but keep images
         await context.route("**/*", lambda route: route.continue_())
 
         return context
@@ -176,7 +174,6 @@ class BrowserlessService:
         except Exception as e:
             logger.warning(f"Navigation warning for {url}: {e}")
 
-        # Wait a bit for dynamic content
         await page.wait_for_timeout(2000)
 
     @staticmethod
@@ -198,71 +195,126 @@ class BrowserlessService:
             pass
 
     @staticmethod
-    async def _wait_for_selector(page: Page, selector: str):
+    async def _extract_amazon_price(page: Page) -> str:
-        """Wait for selector and scroll into view."""
+        """Extract price from Amazon product page."""
+        price_selectors = [
+            ".a-price .a-offscreen",
+            "#corePrice_desktop .a-price .a-offscreen",
+            "#corePriceDisplay_desktop_feature_div .a-price .a-offscreen",
+            ".a-price[data-a-color='price'] .a-offscreen",
+            "#priceblock_ourprice",
+            "#priceblock_dealprice",
+            "span.a-price-whole",
+        ]
+
+        for selector in price_selectors:
+            try:
+                element = page.locator(selector).first
+                price_text = await element.inner_text(timeout=2000)
+                if price_text and price_text.strip():
+                    logger.info(f"Extracted Amazon price via {selector}: {price_text}")
+                    return price_text.strip()
+            except Exception:
+                continue
+
+        # Try combination: whole + fraction
         try:
-            logger.info(f"Waiting for selector: {selector}")
+            whole = await page.locator("span.a-price-whole").first.inner_text()
-            await page.wait_for_selector(selector, timeout=5000)
+            fraction = await page.locator("span.a-price-fraction").first.inner_text()
-            element = page.locator(selector).first
+            if whole and fraction:
-            await element.scroll_into_view_if_needed()
+                price_text = f"{whole}{fraction}"
-            logger.info(f"Scrolled to selector: {selector}")
+                logger.info(f"Extracted Amazon price from whole+fraction: {price_text}")
-        except Exception as e:
+                return price_text
-            logger.warning(f"Selector {selector} not found or timed out: {e}")
+        except Exception:
+            pass
+
+        logger.warning("Could not extract Amazon price")
+        return ""
 
     @staticmethod
-    async def _auto_detect_price(page: Page):
+    async def _extract_generic_price(page: Page) -> str:
-        """Attempt to auto-detect and scroll to price element."""
+        """Extract price from generic e-commerce pages."""
-        logger.info("No selector provided. Attempting to find price element...")
+        price_selectors = [
+            "[itemprop='price']",
+            "[data-testid='price']",
+            "[data-test='price']",
+            ".current-price",
+            ".sale-price",
+            ".final-price",
+            ".product-price",
+            ".special-price",
+            "[data-price]",
+            ".price-current",
+            ".price-now",
+            ".price-sales",
+            "[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='regular']):not([class*='strike']):not([class*='barre'])",
+            "#price",
+            "#product-price",
+            "span[class*='prix']:not([class*='ancien']):not([class*='barre'])",
+            ".prix-actuel",
+            ".price:not(.old-price):not(.was-price)",
+        ]
+
+        found_prices = []
+
+        for selector in price_selectors:
+            try:
+                elements = page.locator(selector)
+                count = await elements.count()
+
+                for i in range(min(count, 3)):
+                    try:
+                        element = elements.nth(i)
+                        if await element.is_visible(timeout=1000):
+                            # Check if strikethrough
+                            try:
+                                text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration")
+                                if "line-through" in text_decoration:
+                                    continue
+                            except Exception:
+                                pass
+
+                            price_text = await element.inner_text()
+                            if price_text and ('€' in price_text or (',' in price_text and any(c.isdigit() for c in price_text))):
+                                price_text = price_text.strip()
+
+                                # Validate price range
+                                numeric_match = re.search(r'(\d+[.,]?\d*)', price_text.replace(' ', '').replace('\xa0', ''))
+                                if numeric_match:
+                                    try:
+                                        price_val = float(numeric_match.group(1).replace(',', '.'))
+                                        if 0.01 <= price_val <= 100000:
+                                            found_prices.append((selector, price_text))
+                                            logger.info(f"Found valid price via {selector}: {price_text}")
+
+                                            # Return high-priority immediately
+                                            if selector in ["[itemprop='price']", "[data-testid='price']", ".current-price", ".sale-price", ".final-price", ".product-price"]:
+                                                return price_text
+                                    except (ValueError, AttributeError):
+                                        continue
+                    except Exception:
+                        continue
+            except Exception:
+                continue
+
+        if found_prices:
+            best_price = found_prices[-1][1]
+            if len(found_prices) > 1:
+                logger.info(f"Multiple prices found, selecting last: {best_price}")
+            return best_price
+
+        # Regex fallback
         try:
-            price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/")
+            all_text = await page.inner_text('body')
-            if await price_locator.count() > 0:
+            price_matches = re.findall(r'\d+[,\.]\d{2}\s*€', all_text)
-                await price_locator.first.scroll_into_view_if_needed()
+            if price_matches:
-                logger.info("Scrolled to potential price element")
+                logger.info(f"Found price via regex: {price_matches[0]}")
-        except Exception as e:
+                return price_matches[0]
-            logger.warning(f"Auto-price detection failed: {e}")
+        except Exception:
+            pass
 
-    @staticmethod
+        logger.warning("Could not extract generic price")
-    async def _smart_scroll(page: Page, scroll_pixels: int):
+        return ""
-        """Perform smart scrolling."""
-        logger.info(f"Performing smart scroll ({scroll_pixels}px)...")
-        try:
-            await page.evaluate(f"window.scrollBy(0, {scroll_pixels})")
-            await page.wait_for_timeout(1000)
-        except Exception as e:
-            logger.warning(f"Smart scroll failed: {e}")
-
-    @staticmethod
-    async def _extract_text(page: Page, text_length: int) -> str:
-        """Extract text from page body."""
-        if text_length <= 0:
-            return ""
-
-        try:
-            logger.info(f"Extracting text (limit: {text_length} chars)...")
-            raw_text = await page.inner_text("body")
-            page_text = raw_text[:text_length]
-            logger.info(f"Extracted {len(page_text)} characters")
-            return page_text
-        except Exception as e:
-            logger.error(f"Text extraction failed: {e}")
-            return ""
-
-    @staticmethod
-    async def _take_screenshot(page: Page, url: str, item_id: int | None = None) -> str:
-        """Take screenshot and save to disk."""
-        screenshot_dir = "/app/screenshots"
-        os.makedirs(screenshot_dir, exist_ok=True)
-
-        if item_id:
-            filename = f"{screenshot_dir}/item_{item_id}.png"
-        else:
-            url_part = url.split("//")[-1].replace("/", "_")
-            timestamp = datetime.now().timestamp()
-            filename = f"{screenshot_dir}/{url_part}_{timestamp}.png"
-
-        await page.screenshot(path=filename, full_page=False)
-        logger.info(f"Screenshot saved to {filename}")
-        return filename
 
     @classmethod
     async def get_page_content(
@@ -270,28 +322,21 @@ class BrowserlessService:
         url: str,
         use_proxy: bool = False,
         wait_selector: str | None = None,
-        config: ScrapeConfig | None = None,
+        extract_text: bool = False
     ) -> tuple[str, str]:
         """
         Fetch page content with full stealth lifecycle.
         
         Args:
-            url: URL to scrape
+            url: URL to fetch
-            use_proxy: Whether to use proxy (reserved for future use)
+            use_proxy: Whether to use proxy
-            wait_selector: Optional CSS selector to wait for
+            wait_selector: CSS selector to wait for
-            config: Optional scraping configuration
+            extract_text: If True, returns visible text; if False, returns HTML
             
         Returns:
-            tuple[screenshot_path, page_text]: Screenshot path and extracted text
+            tuple[content, screenshot_path]: Content (HTML or text) and screenshot path
         """
-        if config is None:
+        # Ensure browser connected
-            config = ScrapeConfig()
-
-        # Input validation & defaults
-        scroll_pixels = max(350, config.scroll_pixels if config.scroll_pixels > 0 else 350)
-        timeout = max(30000, config.timeout if config.timeout > 0 else 90000)
-
-        # Ensure browser is connected and healthy
         if not await cls._ensure_browser_connected():
             logger.error("Failed to establish browser connection")
             return "", ""
@@ -301,28 +346,78 @@ class BrowserlessService:
             page = await context.new_page()
 
             try:
-                await cls._navigate_and_wait(page, url, timeout)
+                await cls._navigate_and_wait(page, url, 90000)
                 await cls._handle_popups(page)
 
-                if wait_selector:
+                # Amazon-specific wait
-                    await cls._wait_for_selector(page, wait_selector)
+                if "amazon" in url.lower() and "/dp/" in url:
+                    amazon_selectors = [".a-price .a-offscreen", "#corePriceDisplay_desktop_feature_div"]
+                    for selector in amazon_selectors:
+                        try:
+                            await page.wait_for_selector(selector, timeout=5000, state="visible")
+                            logger.info(f"Amazon price element found: {selector}")
+                            break
+                        except Exception:
+                            continue
+
+                # Extract content
+                if extract_text:
+                    content = await page.inner_text('body')
+                    logger.info(f"Extracted {len(content)} chars of visible text")
+
+                    # Normalize French prices
+                    content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content)
+                    content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content)
+                    content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content)
+
+                    # Extract price
+                    extracted_price = ""
+                    if "amazon" in url.lower() and "/dp/" in url:
+                        extracted_price = await cls._extract_amazon_price(page)
+                    else:
+                        extracted_price = await cls._extract_generic_price(page)
+
+                    # Prepend normalized price
+                    if extracted_price:
+                        normalized_price = re.sub(r'(\d+)€(\d{2})', r'\1.\2 €', extracted_price)
+                        normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price)
+                        normalized_price = normalized_price.replace(' ', '')
+                        content = f"PRIX DÉTECTÉ: {normalized_price}\n\n{content}"
+                        logger.info(f"Prepended price: {normalized_price}")
                 else:
-                    await cls._auto_detect_price(page)
+                    content = await page.content()
 
-                if config.smart_scroll:
+                # Take screenshot
-                    await cls._smart_scroll(page, scroll_pixels)
+                screenshot_path = ""
+                try:
+                    os.makedirs("/app/screenshots", exist_ok=True)
+                    timestamp = int(time.time() * 1000)
+                    safe_name = "".join(c if c.isalnum() else "_" for c in url.split("//")[-1])[:50]
+                    screenshot_path = f"/app/screenshots/{safe_name}_{timestamp}.jpg"
 
-                page_text = await cls._extract_text(page, config.text_length)
+                    is_amazon = "amazon" in url.lower() and "/dp/" in url
-                
-                # Extract item_id from URL if it matches pattern
-                item_id = None
-                if "item_" in url:
-                    # This is a simplified extraction, adjust based on your URL patterns
-                    pass
-                    
-                screenshot_path = await cls._take_screenshot(page, url, item_id)
 
-                return screenshot_path, page_text
+                    if is_amazon:
+                        # Focused Amazon screenshot
+                        for selector in ["#dp-container", "#ppd", "#centerCol"]:
+                            try:
+                                element = page.locator(selector).first
+                                if await element.is_visible():
+                                    await element.screenshot(path=screenshot_path, quality=85, type="jpeg")
+                                    logger.info(f"Amazon focused screenshot: {selector}")
+                                    break
+                            except Exception:
+                                continue
+                        else:
+                            await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg")
+                    else:
+                        await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg")
+
+                    logger.info(f"Screenshot saved to {screenshot_path}")
+                except Exception as e:
+                    logger.warning(f"Screenshot failed: {e}")
+
+                return content, screenshot_path
 
             finally:
                 await context.close()
@@ -332,5 +427,5 @@ class BrowserlessService:
             return "", ""
 
 
-# Global instance for backward compatibility
+# Global instance
 browserless_service = BrowserlessService()