From ea2af8a8f284e73efa1d49ce1fddbf54db61f660 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Nov 2025 10:34:41 +0000 Subject: [PATCH] feat: Create persistent Amazon scraper service (ScraperService pattern) - New amazon_scraper_service.py with persistent browser connection - Shared browser instance across requests (better session management) - Auto-reconnect if browser connection drops - Better stealth mode (webdriver undefined, chrome runtime) - Proper cookie/popup handling for Amazon - Uses Playwright via Browserless (not Crawl4AI) - Should fix 503 errors with persistent session Based on user's ScraperService pattern for reliability --- app/routers/amazon.py | 6 +- app/services/amazon_scraper_service.py | 419 +++++++++++++++++++++++++ 2 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 app/services/amazon_scraper_service.py diff --git a/app/routers/amazon.py b/app/routers/amazon.py index ac311e0..f79d5e9 100644 --- a/app/routers/amazon.py +++ b/app/routers/amazon.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, Query from fastapi.responses import StreamingResponse from pydantic import BaseModel -from app.services.amazon_scraper_v2 import scrape_amazon_search, AmazonProduct +from app.services.amazon_scraper_service import amazon_scraper_service, AmazonProduct logger = logging.getLogger(__name__) @@ -55,9 +55,9 @@ async def search_amazon( ) yield f"event: progress\ndata: {progress.model_dump_json()}\n\n" - # Execute search + # Execute search with persistent browser service logger.info(f"Starting Amazon search for: {q}") - products = await scrape_amazon_search(q, max_results=max_results) + products = await amazon_scraper_service.scrape_search(q, max_results=max_results) # Convert to dict results = [p.model_dump() for p in products] diff --git a/app/services/amazon_scraper_service.py b/app/services/amazon_scraper_service.py new file mode 100644 index 0000000..b2cbcab --- /dev/null +++ b/app/services/amazon_scraper_service.py @@ -0,0 +1,419 @@ +""" +Amazon Scraper Service - Using persistent browser connection +Based on ScraperService pattern for better session management and anti-detection +""" + +import asyncio +import logging +import os +import re +from dataclasses import dataclass +from datetime import datetime +from urllib.parse import quote_plus + +from bs4 import BeautifulSoup +from playwright.async_api import Browser, BrowserContext, Page, async_playwright +from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") + +# Amazon configuration +AMAZON_FR_BASE_URL = "https://www.amazon.fr" +AMAZON_FR_SEARCH_URL = "https://www.amazon.fr/s?k={query}" + +# Cookie/popup selectors for Amazon +AMAZON_POPUP_SELECTORS = [ + "#sp-cc-accept", # Cookie banner + "#sp-cc-rejectall-link", + "button[data-action='a-popover-close']", + "[data-action='sp-cc-accept']", + "input[aria-labelledby='sp-cc-accept-label']", +] + + +# ============================================================================ +# PYDANTIC SCHEMAS +# ============================================================================ + +class AmazonProduct(BaseModel): + """Schema for Amazon product extraction""" + title: str = Field(description="Product title") + url: str = Field(description="Product URL") + price: float | None = Field(default=None, description="Price in EUR") + original_price: float | None = Field(default=None, description="Original price if discounted") + rating: float | None = Field(default=None, description="Product rating (0-5)") + reviews_count: int | None = Field(default=None, description="Number of reviews") + image_url: str | None = Field(default=None, description="Product image URL") + in_stock: bool = Field(default=True, description="Availability status") + prime: bool = Field(default=False, description="Prime eligible") + sponsored: bool = Field(default=False, description="Is sponsored") + + +# ============================================================================ +# PARSING HELPERS +# ============================================================================ + +def parse_amazon_price(price_text: str) -> float | None: + """Parse Amazon price formats""" + if not price_text: + return None + + cleaned = price_text.strip().replace('€', '').replace('EUR', '').strip() + cleaned = cleaned.replace(' ', '').replace('\xa0', '') + cleaned = cleaned.replace(',', '.') + + match = re.search(r'(\d+\.?\d*)', cleaned) + if match: + try: + return float(match.group(1)) + except ValueError: + return None + return None + + +def parse_rating(rating_text: str) -> float | None: + """Parse rating""" + if not rating_text: + return None + + match = re.search(r'(\d+[,.]\d+)', rating_text) + if match: + try: + return float(match.group(1).replace(',', '.')) + except ValueError: + return None + return None + + +def parse_reviews_count(reviews_text: str) -> int | None: + """Parse review count""" + if not reviews_text: + return None + + cleaned = re.sub(r'[^\d\s]', '', reviews_text) + cleaned = cleaned.replace(' ', '').replace('\xa0', '') + + try: + return int(cleaned) + except ValueError: + return None + + +# ============================================================================ +# AMAZON SCRAPER SERVICE +# ============================================================================ + +class AmazonScraperService: + """Persistent browser service for Amazon scraping""" + + _playwright = None + _browser: Browser | None = None + _lock = asyncio.Lock() + + @classmethod + async def initialize(cls): + """Initialize shared browser (Thread-Safe)""" + async with cls._lock: + await cls._initialize() + + @classmethod + async def _initialize(cls): + """Internal initialization""" + if cls._browser is None: + logger.info("Initializing AmazonScraperService shared browser...") + cls._playwright = await async_playwright().start() + cls._browser = await cls._connect_browser(cls._playwright) + logger.info("AmazonScraperService initialized.") + + @classmethod + async def shutdown(cls): + """Shutdown shared browser""" + async with cls._lock: + if cls._browser: + logger.info("Shutting down AmazonScraperService...") + await cls._browser.close() + cls._browser = None + if cls._playwright: + await cls._playwright.stop() + cls._playwright = None + logger.info("AmazonScraperService shutdown complete.") + + @classmethod + async def _ensure_browser_connected(cls) -> bool: + """Ensure browser is connected, reconnect if needed""" + async with cls._lock: + try: + if cls._browser is None: + logger.warning("Browser not initialized, initializing...") + await cls._initialize() + return cls._browser is not None + + # Test connection + try: + test_context = await cls._browser.new_context() + await test_context.close() + return True + except Exception as e: + logger.error(f"Browser connection test failed: {e}") + logger.info("Attempting to reconnect...") + cls._browser = None + if cls._playwright: + try: + await cls._playwright.stop() + except Exception: + pass + cls._playwright = None + await cls._initialize() + return cls._browser is not None + except Exception as e: + logger.error(f"Failed to ensure browser connection: {e}") + return False + + @staticmethod + async def _connect_browser(p) -> Browser: + """Connect to Browserless""" + logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}") + return await p.chromium.connect_over_cdp(BROWSERLESS_URL) + + @staticmethod + async def _create_context(browser: Browser) -> BrowserContext: + """Create browser context with stealth settings""" + context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ), + locale="fr-FR", + timezone_id="Europe/Paris", + ) + + # Stealth mode + await context.add_init_script(""" + Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + window.chrome = { runtime: {} }; + """) + + await context.route("**/*", lambda route: route.continue_()) + return context + + @staticmethod + async def _handle_popups(page: Page): + """Close Amazon popups/cookies""" + logger.info("Handling Amazon popups...") + for selector in AMAZON_POPUP_SELECTORS: + try: + if await page.locator(selector).count() > 0: + logger.info(f"Found popup: {selector}") + await page.locator(selector).first.click(timeout=2000) + await page.wait_for_timeout(1000) + except Exception: + pass + + try: + await page.keyboard.press("Escape") + except Exception: + pass + + @classmethod + async def scrape_search(cls, query: str, max_results: int = 20) -> list[AmazonProduct]: + """ + Scrape Amazon France search results + + Args: + query: Search query + max_results: Maximum products to return + + Returns: + List of AmazonProduct objects + """ + search_url = AMAZON_FR_SEARCH_URL.format(query=quote_plus(query)) + logger.info(f"🔍 Searching Amazon France: {query}") + logger.info(f"📍 URL: {search_url}") + + # Ensure browser is connected + if not await cls._ensure_browser_connected(): + logger.error("Failed to establish browser connection") + return [] + + products = [] + + try: + context = await cls._create_context(cls._browser) + page = await context.new_page() + + try: + # Navigate + logger.info(f"Navigating to {search_url}") + await page.goto(search_url, wait_until="domcontentloaded", timeout=60000) + logger.info("Page loaded (domcontentloaded)") + + # Wait for network idle + try: + await page.wait_for_load_state("networkidle", timeout=10000) + logger.info("Network idle reached") + except PlaywrightTimeoutError: + logger.info("Network idle timed out (non-critical)") + + # Handle popups + await cls._handle_popups(page) + + # Wait a bit for content + await page.wait_for_timeout(2000) + + # Get HTML + html_content = await page.content() + logger.info(f"✅ Page content extracted ({len(html_content)} bytes)") + + if len(html_content) < 10000: + logger.error(f"❌ Page too small - likely blocked") + return [] + + # Parse with BeautifulSoup + soup = BeautifulSoup(html_content, 'html.parser') + + # Find product cards + product_cards = soup.find_all('div', {'data-component-type': 's-search-result'}) + + if not product_cards: + # Try alternative + product_cards = soup.find_all('div', {'data-asin': True, 'data-index': True}) + + if not product_cards: + logger.warning("⚠️ No products found") + # Check for blocks + if '503' in html_content or 'robot' in html_content.lower(): + logger.error("🚫 Amazon blocked request") + return [] + + logger.info(f"📦 Found {len(product_cards)} product cards") + + # Extract products + for idx, card in enumerate(product_cards): + if len(products) >= max_results: + break + + try: + product = cls._extract_product(card, idx) + if product: + products.append(product) + logger.debug(f" ✓ [{len(products)}] {product.title[:50]}... - {product.price}€") + except Exception as e: + logger.error(f"Error parsing card {idx}: {e}") + continue + + logger.info(f"✅ Successfully extracted {len(products)} products") + + finally: + await context.close() + + except Exception as e: + logger.error(f"❌ Error during scraping: {e}", exc_info=True) + return [] + + return products + + @staticmethod + def _extract_product(card, idx: int) -> AmazonProduct | None: + """Extract product data from card""" + # ASIN + asin = card.get('data-asin', '') + if not asin: + logger.debug(f" ⏭️ Card {idx}: No ASIN") + return None + + # Sponsored + sponsored = bool(card.select_one('[data-component-type="sp-sponsored-result"]')) + + # Title + title = None + for selector in ['h2 a span', 'h2 span', 'h2.s-line-clamp-2 span']: + elem = card.select_one(selector) + if elem: + title = elem.get_text(strip=True) + if title: + break + + if not title: + logger.debug(f" ⏭️ Card {idx}: No title") + return None + + # URL + link_elem = card.select_one('h2 a') or card.select_one('a.s-link-style') + if not link_elem: + logger.debug(f" ⏭️ Card {idx}: No link") + return None + + href = link_elem.get('href', '') + product_url = f"{AMAZON_FR_BASE_URL}{href}" if href.startswith('/') else href + + # Price + price = None + for selector in ['.a-price .a-offscreen', '.a-price-whole', 'span.a-price span.a-offscreen']: + elem = card.select_one(selector) + if elem: + price = parse_amazon_price(elem.get_text(strip=True)) + if price: + break + + # Original price + original_price = None + elem = card.select_one('.a-price.a-text-price .a-offscreen') + if elem: + original_price = parse_amazon_price(elem.get_text(strip=True)) + + # Rating + rating = None + for selector in ['[aria-label*="étoile"]', '[aria-label*="star"]']: + elem = card.select_one(selector) + if elem: + rating = parse_rating(elem.get('aria-label', '')) + if rating: + break + + # Reviews + reviews_count = None + for selector in ['[aria-label*="étoile"] + span', 'span.s-underline-text']: + elem = card.select_one(selector) + if elem: + reviews_count = parse_reviews_count(elem.get_text(strip=True)) + if reviews_count: + break + + # Image + image_url = None + for selector in ['img.s-image', 'img']: + elem = card.select_one(selector) + if elem: + image_url = elem.get('src') or elem.get('data-src') + if image_url: + break + + # Prime + prime = bool(card.select_one('[aria-label*="Prime"]') or card.select_one('i.a-icon-prime')) + + # Stock + in_stock = True + if card.select_one('[aria-label*="Indisponible"]'): + in_stock = False + + return AmazonProduct( + title=title, + url=product_url, + price=price, + original_price=original_price, + rating=rating, + reviews_count=reviews_count, + image_url=image_url, + in_stock=in_stock, + prime=prime, + sponsored=sponsored, + ) + + +# Global instance +amazon_scraper_service = AmazonScraperService()