feat: Add Playwright-based web scraping service with advanced stealth techniques and proxy support, including Amazon-specific logic.

app/services/direct_search_service.py

@@ -41,18 +41,14 @@ USER_AGENT_POOL = [
     # Chrome Windows - Latest versions
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
     # Chrome Mac
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
     # Firefox Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
-    # Firefox Mac
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
     # Edge Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    # Linux
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 ]
 
 # Amazon-specific blocking indicators
@@ -992,6 +988,18 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             if is_amazon:
                 await _random_delay(1.0, 3.0)
 
+            # Warmup for Amazon: Visit home page first to get cookies (only on first attempt)
+            if is_amazon and attempt == 0:
+                try:
+                    logger.info(f"Amazon Warmup: Visiting home page...")
+                    # Use a shorter timeout for warmup
+                    await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
+                    await asyncio.sleep(random.uniform(2.0, 4.0))
+                    # Simulate a bit of mouse movement
+                    await _simulate_human_behavior(page)
+                except Exception as e:
+                    logger.warning(f"Amazon warmup failed (non-critical): {e}")
+
             # Naviguer vers la page de recherche
             await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
 

app/services/scraper_service.py

@@ -40,6 +40,7 @@ USER_AGENT_POOL = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 ]
 
 # Amazon blocking indicators
@@ -564,6 +565,14 @@ class ScraperService:
                 # Add random delay before navigation for Amazon
                 if is_amazon:
                     await asyncio.sleep(random.uniform(1.0, 3.0))
+                    
+                    # Warmup: Visit home page first
+                    try:
+                        logger.info("Amazon Warmup: Visiting home page...")
+                        await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
+                        await asyncio.sleep(random.uniform(2.0, 4.0))
+                    except Exception as e:
+                        logger.warning(f"Amazon warmup failed: {e}")
 
                 logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
                 response = None

debug_amazon.py

@@ -0,0 +1,60 @@
+import asyncio
+import logging
+import os
+from playwright.async_api import async_playwright
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
+
+async def debug_amazon():
+    logger.info("Starting Amazon Debug Script")
+    
+    async with async_playwright() as p:
+        try:
+            logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+            browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
+            
+            # Use a very standard, recent User-Agent
+            user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            
+            context = await browser.new_context(
+                user_agent=user_agent,
+                viewport={"width": 1920, "height": 1080},
+                locale="fr-FR",
+                timezone_id="Europe/Paris"
+            )
+            
+            page = await context.new_page()
+            
+            url = "https://www.amazon.fr/s?k=iphone"
+            logger.info(f"Navigating to {url}")
+            
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            
+            if response:
+                status = response.status
+                logger.info(f"Response Status: {status}")
+                
+                content = await page.content()
+                if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content:
+                    logger.error("BLOCK DETECTED: Found blocking message in content")
+                else:
+                    logger.info("No obvious blocking message found")
+                    
+                # Save screenshot
+                await page.screenshot(path="debug_amazon_screenshot.png")
+                logger.info("Screenshot saved to debug_amazon_screenshot.png")
+                
+            else:
+                logger.error("No response received")
+                
+            await browser.close()
+            
+        except Exception as e:
+            logger.error(f"An error occurred: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(debug_amazon())