From 3caea8316b1e5cbd88884ab107d4bdc1fbeff230 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Fri, 28 Nov 2025 23:53:45 +0100 Subject: [PATCH] feat: Add Playwright-based web scraping service with advanced stealth techniques and proxy support, including Amazon-specific logic. --- app/services/direct_search_service.py | 20 ++++++--- app/services/scraper_service.py | 9 ++++ debug_amazon.py | 60 +++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 debug_amazon.py diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py index 5728373..d2ea58f 100644 --- a/app/services/direct_search_service.py +++ b/app/services/direct_search_service.py @@ -41,18 +41,14 @@ USER_AGENT_POOL = [ # Chrome Windows - Latest versions "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", # Chrome Mac "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", # Firefox Windows "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0", - # Firefox Mac - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0", # Edge Windows "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", + # Linux + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", ] # Amazon-specific blocking indicators @@ -992,6 +988,18 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 if is_amazon: await _random_delay(1.0, 3.0) + # Warmup for Amazon: Visit home page first to get cookies (only on first attempt) + if is_amazon and attempt == 0: + try: + logger.info(f"Amazon Warmup: Visiting home page...") + # Use a shorter timeout for warmup + await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000) + await asyncio.sleep(random.uniform(2.0, 4.0)) + # Simulate a bit of mouse movement + await _simulate_human_behavior(page) + except Exception as e: + logger.warning(f"Amazon warmup failed (non-critical): {e}") + # Naviguer vers la page de recherche await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000)) diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py index 7ffd465..b36551b 100644 --- a/app/services/scraper_service.py +++ b/app/services/scraper_service.py @@ -40,6 +40,7 @@ USER_AGENT_POOL = [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", ] # Amazon blocking indicators @@ -564,6 +565,14 @@ class ScraperService: # Add random delay before navigation for Amazon if is_amazon: await asyncio.sleep(random.uniform(1.0, 3.0)) + + # Warmup: Visit home page first + try: + logger.info("Amazon Warmup: Visiting home page...") + await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000) + await asyncio.sleep(random.uniform(2.0, 4.0)) + except Exception as e: + logger.warning(f"Amazon warmup failed: {e}") logger.info(f"Navigating to {url} (Timeout: {timeout}ms)") response = None diff --git a/debug_amazon.py b/debug_amazon.py new file mode 100644 index 0000000..2497fbf --- /dev/null +++ b/debug_amazon.py @@ -0,0 +1,60 @@ +import asyncio +import logging +import os +from playwright.async_api import async_playwright + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") + +async def debug_amazon(): + logger.info("Starting Amazon Debug Script") + + async with async_playwright() as p: + try: + logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}") + browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL) + + # Use a very standard, recent User-Agent + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + + context = await browser.new_context( + user_agent=user_agent, + viewport={"width": 1920, "height": 1080}, + locale="fr-FR", + timezone_id="Europe/Paris" + ) + + page = await context.new_page() + + url = "https://www.amazon.fr/s?k=iphone" + logger.info(f"Navigating to {url}") + + response = await page.goto(url, wait_until="domcontentloaded", timeout=30000) + + if response: + status = response.status + logger.info(f"Response Status: {status}") + + content = await page.content() + if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content: + logger.error("BLOCK DETECTED: Found blocking message in content") + else: + logger.info("No obvious blocking message found") + + # Save screenshot + await page.screenshot(path="debug_amazon_screenshot.png") + logger.info("Screenshot saved to debug_amazon_screenshot.png") + + else: + logger.error("No response received") + + await browser.close() + + except Exception as e: + logger.error(f"An error occurred: {e}") + +if __name__ == "__main__": + asyncio.run(debug_amazon())