From c6b9befc9a62d72e3d3db33fc91b969757b5c635 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 13:25:13 +0100 Subject: [PATCH] feat: introduce BrowserlessService for persistent, auto-reconnecting browser operations and enhanced price extraction --- app/services/browserless_service.py | 237 ++++++++++++++++++---------- 1 file changed, 150 insertions(+), 87 deletions(-) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 9ed981b..9e77a6d 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -1,10 +1,13 @@ """ -Browserless Service - Enhanced Version -Manages connections to Browserless.io with advanced robustness features: -- Auto-reconnection on browser disconnection -- Advanced popup handling -- Flexible configuration -- Thread-safe operations +Browserless Service - Enhanced Persistent Browser Version +Inspired by AmazonScraperService architecture for better performance and reliability + +Features: +- Persistent browser connection (singleton pattern) +- Auto-reconnection on disconnection +- Thread-safe operations (asyncio.Lock) +- Improved price extraction accuracy +- Optimized timeouts and screenshots - Amazon & Generic price extraction """ @@ -45,6 +48,7 @@ POPUP_SELECTORS = [ "#onetrust-accept-btn-handler", "button:has-text('Tout accepter')", "button:has-text('Accepter')", + "input[aria-labelledby='sp-cc-accept-label']", ] @@ -55,11 +59,14 @@ class ScrapeConfig: smart_scroll: bool = False scroll_pixels: int = 350 text_length: int = 0 - timeout: int = 90000 + timeout: int = 30000 # Reduced from 90s to 30s class BrowserlessService: - """Enhanced browserless service with auto-reconnection and robust error handling.""" + """ + Enhanced persistent browserless service with auto-reconnection. + Architecture inspired by AmazonScraperService for optimal performance. + """ _playwright = None _browser: Browser | None = None @@ -75,10 +82,10 @@ class BrowserlessService: async def _initialize(cls): """Internal initialization logic (Assumes lock is held).""" if cls._browser is None: - logger.info("Initializing BrowserlessService shared browser...") + logger.info("πŸš€ Initializing BrowserlessService persistent browser...") cls._playwright = await async_playwright().start() cls._browser = await cls._connect_browser(cls._playwright) - logger.info("BrowserlessService initialized.") + logger.info("βœ… BrowserlessService browser ready (persistent)") @classmethod async def shutdown(cls): @@ -95,7 +102,10 @@ class BrowserlessService: @classmethod async def _ensure_browser_connected(cls) -> bool: - """Ensure browser is connected, reconnect if needed.""" + """ + Ensure browser is connected, reconnect if needed. + Returns True if browser is ready, False otherwise. + """ async with cls._lock: try: if cls._browser is None: @@ -107,10 +117,13 @@ class BrowserlessService: try: test_context = await cls._browser.new_context() await test_context.close() + logger.debug("βœ… Browser connection test passed (reusing)") return True except Exception as e: - logger.error(f"Browser connection test failed: {e}") - logger.info("Attempting to reconnect browser...") + logger.error(f"❌ Browser connection test failed: {e}") + logger.info("πŸ”„ Attempting to reconnect browser...") + + # Clear the old browser cls._browser = None if cls._playwright: try: @@ -118,6 +131,8 @@ class BrowserlessService: except Exception: pass cls._playwright = None + + # Reconnect await cls._initialize() return cls._browser is not None except Exception as e: @@ -160,32 +175,32 @@ class BrowserlessService: @staticmethod async def _navigate_and_wait(page: Page, url: str, timeout: int): """Navigate to URL and wait for page load.""" - logger.info(f"Navigating to {url} (Timeout: {timeout}ms)") + logger.info(f"πŸ“‘ Navigating to {url} (Timeout: {timeout}ms)") try: await page.goto(url, wait_until="domcontentloaded", timeout=timeout) - logger.info(f"Page loaded (domcontentloaded): {url}") + logger.info(f"βœ… Page loaded (domcontentloaded): {url}") try: - await page.wait_for_load_state("networkidle", timeout=5000) - logger.info("Network idle reached") + await page.wait_for_load_state("networkidle", timeout=2000) # Reduced from 5s + logger.info("βœ… Network idle reached") except PlaywrightTimeoutError: - logger.info("Network idle timed out (non-critical), proceeding...") + logger.debug("⚠️ Network idle timed out (non-critical), proceeding...") except Exception as e: logger.warning(f"Navigation warning for {url}: {e}") - await page.wait_for_timeout(2000) + await page.wait_for_timeout(1500) # Reduced from 2s @staticmethod async def _handle_popups(page: Page): """Attempt to close popups and cookie banners.""" - logger.info("Attempting to close popups...") + logger.debug("Attempting to close popups...") for popup_selector in POPUP_SELECTORS: try: if await page.locator(popup_selector).count() > 0: - logger.info(f"Found popup close button: {popup_selector}") + logger.info(f"🚫 Closing popup: {popup_selector}") await page.locator(popup_selector).first.click(timeout=2000) - await page.wait_for_timeout(1000) + await page.wait_for_timeout(500) except Exception: pass @@ -196,9 +211,9 @@ class BrowserlessService: @staticmethod async def _extract_amazon_price(page: Page) -> str: - """Extract price from Amazon product page.""" + """Extract price from Amazon product page with strict validation.""" price_selectors = [ - ".a-price .a-offscreen", + ".a-price .a-offscreen", # Most reliable "#corePrice_desktop .a-price .a-offscreen", "#corePriceDisplay_desktop_feature_div .a-price .a-offscreen", ".a-price[data-a-color='price'] .a-offscreen", @@ -212,8 +227,10 @@ class BrowserlessService: element = page.locator(selector).first price_text = await element.inner_text(timeout=2000) if price_text and price_text.strip(): - logger.info(f"Extracted Amazon price via {selector}: {price_text}") - return price_text.strip() + # Validate format (Amazon uses "XX,YY €" or "XX,YY") + if re.search(r'\d+[.,]\d{2}', price_text): + logger.info(f"πŸ’° Amazon price via {selector}: {price_text}") + return price_text.strip() except Exception: continue @@ -223,97 +240,138 @@ class BrowserlessService: fraction = await page.locator("span.a-price-fraction").first.inner_text() if whole and fraction: price_text = f"{whole}{fraction}" - logger.info(f"Extracted Amazon price from whole+fraction: {price_text}") + logger.info(f"πŸ’° Amazon price from whole+fraction: {price_text}") return price_text except Exception: pass - logger.warning("Could not extract Amazon price") + logger.warning("⚠️ Could not extract Amazon price") return "" @staticmethod async def _extract_generic_price(page: Page) -> str: - """Extract price from generic e-commerce pages.""" - price_selectors = [ + """ + Extract price from generic e-commerce pages with STRICT validation. + Improvements: + - Strict French price regex: digits,digits € + - Context validation (avoid random numbers) + - Exclude strikethrough prices + """ + # High-priority semantic selectors + high_priority_selectors = [ "[itemprop='price']", "[data-testid='price']", "[data-test='price']", + "[data-price]", + ] + + # Medium-priority specific selectors + medium_priority_selectors = [ ".current-price", ".sale-price", ".final-price", ".product-price", ".special-price", - "[data-price]", ".price-current", ".price-now", - ".price-sales", - "[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='regular']):not([class*='strike']):not([class*='barre'])", + ".prix-actuel", "#price", "#product-price", - "span[class*='prix']:not([class*='ancien']):not([class*='barre'])", - ".prix-actuel", - ".price:not(.old-price):not(.was-price)", ] + # Low-priority generic selectors + low_priority_selectors = [ + "span[class*='prix']:not([class*='ancien']):not([class*='barre'])", + ".price:not(.old-price):not(.was-price)", + "[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='strike']):not([class*='barre'])", + ] + + all_selectors = high_priority_selectors + medium_priority_selectors + low_priority_selectors found_prices = [] - for selector in price_selectors: + # STRICT French price regex: 1-4 digits, comma, 2 digits, optional € symbol + # Examples: "12,99 €", "1 234,99€", "3,50 EUR" + strict_price_regex = re.compile(r'(\d{1,4}(?:\s?\d{3})*[.,]\d{2})\s*€?') + + for selector in all_selectors: try: elements = page.locator(selector) count = await elements.count() - for i in range(min(count, 3)): + for i in range(min(count, 3)): # Max 3 elements per selector try: element = elements.nth(i) - if await element.is_visible(timeout=1000): - # Check if strikethrough + + # Check visibility + if not await element.is_visible(timeout=1000): + continue + + # Check if strikethrough (old price) + try: + text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration") + if "line-through" in text_decoration: + logger.debug(f"⏭️ Skipping strikethrough price at {selector}") + continue + except Exception: + pass + + price_text = await element.inner_text() + if not price_text: + continue + + # STRICT validation with regex + price_match = strict_price_regex.search(price_text) + if price_match: + matched_price = price_match.group(0) + + # Extract numeric value for range validation + numeric_text = matched_price.replace('€', '').replace(' ', '').replace(',', '.') try: - text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration") - if "line-through" in text_decoration: - continue - except Exception: - pass + price_val = float(numeric_text) + + # Reasonable price range: 0.01€ to 100,000€ + if 0.01 <= price_val <= 100000: + found_prices.append((selector, matched_price)) + logger.info(f"πŸ’° Valid price via {selector}: {matched_price} ({price_val}€)") - price_text = await element.inner_text() - if price_text and ('€' in price_text or (',' in price_text and any(c.isdigit() for c in price_text))): - price_text = price_text.strip() + # Return immediately if high-priority selector + if selector in high_priority_selectors: + return matched_price + else: + logger.debug(f"⏭️ Price out of range ({price_val}€): {matched_price}") + except ValueError: + logger.debug(f"⏭️ Could not parse price value: {matched_price}") + continue - # Validate price range - numeric_match = re.search(r'(\d+[.,]?\d*)', price_text.replace(' ', '').replace('\xa0', '')) - if numeric_match: - try: - price_val = float(numeric_match.group(1).replace(',', '.')) - if 0.01 <= price_val <= 100000: - found_prices.append((selector, price_text)) - logger.info(f"Found valid price via {selector}: {price_text}") - - # Return high-priority immediately - if selector in ["[itemprop='price']", "[data-testid='price']", ".current-price", ".sale-price", ".final-price", ".product-price"]: - return price_text - except (ValueError, AttributeError): - continue - except Exception: + except Exception as e: + logger.debug(f"Error processing element {i} of {selector}: {e}") continue - except Exception: + except Exception as e: + logger.debug(f"Error with selector {selector}: {e}") continue + # If multiple prices found, prefer the FIRST valid one (usually current price) + # Changed from LAST to FIRST - more reliable for current price detection if found_prices: - best_price = found_prices[-1][1] + best_price = found_prices[0][1] # Take first (highest priority) if len(found_prices) > 1: - logger.info(f"Multiple prices found, selecting last: {best_price}") + prices_list = [p[1] for p in found_prices] + logger.info(f"πŸ’° Multiple prices found: {prices_list}, selecting first: {best_price}") return best_price - # Regex fallback + # Fallback: Strict regex search in body text (last resort) try: all_text = await page.inner_text('body') - price_matches = re.findall(r'\d+[,\.]\d{2}\s*€', all_text) + price_matches = strict_price_regex.findall(all_text) if price_matches: - logger.info(f"Found price via regex: {price_matches[0]}") - return price_matches[0] - except Exception: - pass + # Take first match + first_match = price_matches[0] + logger.info(f"πŸ’° Found price via regex fallback: {first_match}") + return first_match + except Exception as e: + logger.debug(f"Regex fallback failed: {e}") - logger.warning("Could not extract generic price") + logger.warning("⚠️ Could not extract generic price with any method") return "" @classmethod @@ -325,7 +383,7 @@ class BrowserlessService: extract_text: bool = False ) -> tuple[str, str]: """ - Fetch page content with full stealth lifecycle. + Fetch page content with persistent browser and optimized performance. Args: url: URL to fetch @@ -336,9 +394,9 @@ class BrowserlessService: Returns: tuple[content, screenshot_path]: Content (HTML or text) and screenshot path """ - # Ensure browser connected + # Ensure browser connected (with auto-reconnection) if not await cls._ensure_browser_connected(): - logger.error("Failed to establish browser connection") + logger.error("❌ Failed to establish browser connection") return "", "" try: @@ -346,7 +404,8 @@ class BrowserlessService: page = await context.new_page() try: - await cls._navigate_and_wait(page, url, 90000) + # Navigate with optimized timeout (30s instead of 90s) + await cls._navigate_and_wait(page, url, 30000) await cls._handle_popups(page) # Amazon-specific wait @@ -355,7 +414,7 @@ class BrowserlessService: for selector in amazon_selectors: try: await page.wait_for_selector(selector, timeout=5000, state="visible") - logger.info(f"Amazon price element found: {selector}") + logger.info(f"βœ… Amazon price element found: {selector}") break except Exception: continue @@ -363,14 +422,14 @@ class BrowserlessService: # Extract content if extract_text: content = await page.inner_text('body') - logger.info(f"Extracted {len(content)} chars of visible text") + logger.info(f"πŸ“„ Extracted {len(content)} chars of visible text") - # Normalize French prices + # Normalize French prices to English format for AI content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content) content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content) - # Extract price + # Extract price with improved accuracy extracted_price = "" if "amazon" in url.lower() and "/dp/" in url: extracted_price = await cls._extract_amazon_price(page) @@ -383,11 +442,14 @@ class BrowserlessService: normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price) normalized_price = normalized_price.replace(' ', '') content = f"PRIX DΓ‰TECTΓ‰: {normalized_price}\n\n{content}" - logger.info(f"Prepended price: {normalized_price}") + logger.info(f"πŸ’° Prepended price: {normalized_price}") + else: + logger.warning("⚠️ No price detected for this page") else: content = await page.content() + logger.debug(f"πŸ“„ Extracted {len(content)} chars of HTML") - # Take screenshot + # Take screenshot (optimized - viewport only by default) screenshot_path = "" try: os.makedirs("/app/screenshots", exist_ok=True) @@ -404,16 +466,17 @@ class BrowserlessService: element = page.locator(selector).first if await element.is_visible(): await element.screenshot(path=screenshot_path, quality=85, type="jpeg") - logger.info(f"Amazon focused screenshot: {selector}") + logger.info(f"πŸ“Έ Amazon focused screenshot: {selector}") break except Exception: continue else: await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") else: + # Viewport screenshot (faster than full_page) await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") + logger.info(f"πŸ“Έ Viewport screenshot saved to {screenshot_path}") - logger.info(f"Screenshot saved to {screenshot_path}") except Exception as e: logger.warning(f"Screenshot failed: {e}") @@ -423,9 +486,9 @@ class BrowserlessService: await context.close() except Exception as e: - logger.error(f"Error scraping {url}: {e}") + logger.error(f"❌ Error scraping {url}: {e}", exc_info=True) return "", "" -# Global instance +# Global instance (backward compatibility) browserless_service = BrowserlessService()