From a8ff8217b353ee8ec5d0604ebe23ec3666e93910 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 07:27:45 +0000
Subject: [PATCH 01/13] fix: Use absolute paths for debug_dumps directory

- Changed debug_dumps path from relative to absolute (/app/debug_dumps)
- Updated direct_search_service.py to use consistent absolute path
- Updated debug.py router to use the same absolute path
- Added debug_dumps directory creation in Dockerfile
- Improved _dump_debug_html with safer filename handling
---
 Dockerfile                            |  4 ++--
 app/routers/debug.py                  |  3 ++-
 app/services/direct_search_service.py | 22 +++++++++++++++++-----
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2532557..e030d81 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -37,8 +37,8 @@ RUN chmod +x docker-entrypoint.sh
 # Copy built frontend static files
 COPY --from=frontend-build /app/frontend/dist /app/static
 
-# Create screenshots directory
-RUN mkdir -p screenshots
+# Create screenshots and debug_dumps directories
+RUN mkdir -p screenshots debug_dumps
 
 # Expose port
 EXPOSE 8555
diff --git a/app/routers/debug.py b/app/routers/debug.py
index 07d694b..54a9b25 100644
--- a/app/routers/debug.py
+++ b/app/routers/debug.py
@@ -18,7 +18,8 @@ router = APIRouter(
 
 logger = logging.getLogger(__name__)
 
-DEBUG_DIR = "debug_dumps"
+# Chemin absolu pour les dumps de débogage (doit correspondre à celui dans direct_search_service.py)
+DEBUG_DIR = "/app/debug_dumps"
 
 class DebugFile(BaseModel):
     filename: str
diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 4abc547..17622b5 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -7,6 +7,7 @@ import asyncio
 import logging
 import os
 import re
+import time
 from urllib.parse import quote_plus, urljoin, urlparse
 
 from bs4 import BeautifulSoup
@@ -25,19 +26,30 @@ USER_AGENT = (
 )
 
 
+# Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py)
+DEBUG_DUMPS_DIR = "/app/debug_dumps"
+
+
 def _dump_debug_html(html: str, domain: str, query: str):
     """Sauvegarde le HTML pour débogage"""
     try:
-        debug_dir = "debug_dumps"
-        if not os.path.exists(debug_dir):
-            os.makedirs(debug_dir)
-            
-        filename = f"{debug_dir}/{domain}_{query}_{int(asyncio.get_event_loop().time())}.html"
+        if not os.path.exists(DEBUG_DUMPS_DIR):
+            os.makedirs(DEBUG_DUMPS_DIR, exist_ok=True)
+            logger.info(f"Created debug dumps directory: {DEBUG_DUMPS_DIR}")
+
+        # Utiliser un timestamp unique basé sur le temps actuel
+        timestamp = int(time.time() * 1000)
+        # Nettoyer la query pour le nom de fichier
+        safe_query = "".join(c if c.isalnum() else "_" for c in query)[:50]
+        filename = f"{DEBUG_DUMPS_DIR}/{domain}_{safe_query}_{timestamp}.html"
+
         with open(filename, "w", encoding="utf-8") as f:
             f.write(html)
         logger.info(f"HTML dump saved to {filename}")
+        return filename
     except Exception as e:
         logger.error(f"Failed to dump HTML: {e}")
+        return None
 
 
 class SearchResult:

From ffeeafa3182d2687fd2ed926541aa113542f5453 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 07:32:00 +0000
Subject: [PATCH 02/13] fix: Add missing debug tab rendering in Admin page

The debug tab was defined in the tabs array and had all the state and
functions set up, but was missing the actual JSX rendering block.
This caused the debug dumps to not be displayed in the admin interface.
---
 frontend/src/pages/Admin.jsx | 48 ++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx
index 2ffe8d8..abb631b 100644
--- a/frontend/src/pages/Admin.jsx
+++ b/frontend/src/pages/Admin.jsx
@@ -1018,6 +1018,54 @@ export default function Admin() {
                     </CardContent>
                 </Card>
             )}
+
+            {/* Debug Tab */}
+            {activeTab === 'debug' && (
+                <Card>
+                    <CardHeader>
+                        <div className="flex items-center justify-between">
+                            <div>
+                                <CardTitle className="flex items-center gap-2"><Bug className="h-5 w-5" />Debug HTML Dumps</CardTitle>
+                                <CardDescription>Fichiers HTML sauvegardés lors des recherches sans résultats.</CardDescription>
+                            </div>
+                            <Button variant="outline" size="sm" onClick={fetchDebugFiles} disabled={loadingDebugFiles}>
+                                <RefreshCw className={cn("h-4 w-4 mr-2", loadingDebugFiles && "animate-spin")} />
+                                Actualiser
+                            </Button>
+                        </div>
+                    </CardHeader>
+                    <CardContent>
+                        {loadingDebugFiles ? (
+                            <div className="text-center py-8 text-muted-foreground">Chargement...</div>
+                        ) : debugFiles.length === 0 ? (
+                            <div className="text-center py-8 text-muted-foreground">
+                                Aucun fichier de debug disponible.
+                            </div>
+                        ) : (
+                            <div className="space-y-2">
+                                {debugFiles.map(file => (
+                                    <div key={file.filename} className="flex items-center justify-between p-3 rounded-lg border bg-card hover:bg-muted/50 transition-colors">
+                                        <div className="flex-1 min-w-0">
+                                            <p className="font-medium text-sm truncate">{file.filename}</p>
+                                            <p className="text-xs text-muted-foreground">
+                                                {(file.size / 1024).toFixed(1)} KB • {file.created_at_formatted}
+                                            </p>
+                                        </div>
+                                        <div className="flex items-center gap-1 ml-4">
+                                            <Button variant="ghost" size="icon" className="h-8 w-8" onClick={() => downloadDebugFile(file.filename)} title="Télécharger">
+                                                <Download className="h-4 w-4" />
+                                            </Button>
+                                            <Button variant="ghost" size="icon" className="h-8 w-8 text-destructive hover:text-destructive" onClick={() => deleteDebugFile(file.filename)} title="Supprimer">
+                                                <Trash2 className="h-4 w-4" />
+                                            </Button>
+                                        </div>
+                                    </div>
+                                ))}
+                            </div>
+                        )}
+                    </CardContent>
+                </Card>
+            )}
         </div >
     );
 }

From 7243bcfaa1aa874c06892d1341f34f408de0e91e Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 07:45:09 +0000
Subject: [PATCH 03/13] fix: Improve Amazon scraping to avoid 503 blocking
 errors

- Add User-Agent rotation pool with modern Chrome/Firefox/Edge versions
- Implement Amazon-specific blocking detection ("Toutes nos excuses", etc.)
- Add exponential backoff with jitter for Amazon retries (up to 4 retries)
- Include Sec-Fetch-* headers and proper Referer to appear more legitimate
- Simulate human behavior (mouse movements, random scrolling)
- Add random delays between requests to avoid rate limiting
- Properly differentiate Amazon 503 (blocking) from true product unavailability
- Apply same protections to both search and product scraping services
---
 app/services/direct_search_service.py | 396 ++++++++++++++++++--------
 app/services/scraper_service.py       | 147 ++++++++--
 2 files changed, 407 insertions(+), 136 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 17622b5..995c39f 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -6,6 +6,7 @@ Utilise Playwright/Browserless pour gérer JavaScript et contourner la détectio
 import asyncio
 import logging
 import os
+import random
 import re
 import time
 from urllib.parse import quote_plus, urljoin, urlparse
@@ -20,10 +21,92 @@ BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 # Constants
 MIN_TITLE_LENGTH = 3
 MIN_LINK_TEXT_LENGTH = 5
-USER_AGENT = (
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
-)
+
+# Pool of realistic User-Agents for rotation (Chrome on Windows/Mac)
+USER_AGENT_POOL = [
+    # Chrome Windows - Latest versions
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
+    # Chrome Mac
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+    # Firefox Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
+    # Firefox Mac
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
+    # Edge Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+]
+
+# Amazon-specific blocking indicators
+AMAZON_BLOCK_INDICATORS = [
+    "Toutes nos excuses",
+    "Sorry, something went wrong",
+    "To discuss automated access to Amazon data",
+    "api-services-support@amazon.com",
+    "Enter the characters you see below",
+    "Saisissez les caractères",
+    "Type the characters you see",
+    "Sorry! Something went wrong",
+    "Service Unavailable",
+    "robot check",
+    "automated access",
+    "ref=cs_503",
+]
+
+# Amazon retry configuration
+AMAZON_MAX_RETRIES = 4
+AMAZON_BASE_DELAY = 3.0  # seconds
+AMAZON_MAX_DELAY = 30.0  # seconds
+
+
+def _get_random_user_agent() -> str:
+    """Get a random User-Agent from the pool"""
+    return random.choice(USER_AGENT_POOL)
+
+
+def _is_amazon_blocked(html_content: str) -> bool:
+    """Check if Amazon has blocked the request"""
+    html_lower = html_content.lower()
+    for indicator in AMAZON_BLOCK_INDICATORS:
+        if indicator.lower() in html_lower:
+            return True
+    return False
+
+
+async def _random_delay(min_seconds: float = 1.0, max_seconds: float = 3.0):
+    """Add a random delay to simulate human behavior"""
+    delay = random.uniform(min_seconds, max_seconds)
+    await asyncio.sleep(delay)
+
+
+async def _simulate_human_behavior(page):
+    """Simulate human-like behavior on the page"""
+    try:
+        # Random mouse movements
+        for _ in range(random.randint(2, 4)):
+            x = random.randint(100, 1800)
+            y = random.randint(100, 900)
+            await page.mouse.move(x, y)
+            await asyncio.sleep(random.uniform(0.1, 0.3))
+
+        # Small random scroll
+        scroll_amount = random.randint(100, 400)
+        await page.evaluate(f"window.scrollBy(0, {scroll_amount})")
+        await asyncio.sleep(random.uniform(0.3, 0.7))
+
+        # Scroll back up a bit
+        await page.evaluate(f"window.scrollBy(0, -{random.randint(50, 150)})")
+        await asyncio.sleep(random.uniform(0.2, 0.5))
+    except Exception as e:
+        logger.debug(f"Human behavior simulation error (non-critical): {e}")
+
+
+# Legacy constant for backward compatibility
+USER_AGENT = USER_AGENT_POOL[0]
 
 
 # Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py)
@@ -562,142 +645,229 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
 
     logger.info(f"Recherche Browserless sur {domain}: {final_url}")
 
-    context = None
-    page = None
+    # Check if this is Amazon - requires special handling
+    is_amazon = "amazon" in domain.lower()
+    max_retries = AMAZON_MAX_RETRIES if is_amazon else 1
 
-    try:
-        # Créer un nouveau contexte isolé pour ce site
-        context = await browser.new_context(
-            viewport={"width": 1920, "height": 1080},
-            user_agent=USER_AGENT,
-            locale="fr-FR",
-        )
-        
-        # Bloquer les ressources inutiles
-        await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
-        await context.route("**/analytics*", lambda route: route.abort())
-        await context.route("**/tracking*", lambda route: route.abort())
-        
-        page = await context.new_page()
+    for attempt in range(max_retries):
+        if attempt > 0:
+            # Exponential backoff with jitter for Amazon retries
+            base_delay = AMAZON_BASE_DELAY * (2 ** attempt)
+            jitter = random.uniform(0.5, 1.5)
+            delay = min(base_delay * jitter, AMAZON_MAX_DELAY)
+            logger.info(f"Amazon retry {attempt + 1}/{max_retries} for '{query}' after {delay:.1f}s delay")
+            await asyncio.sleep(delay)
 
-        # Naviguer vers la page de recherche
-        await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
+        context = None
+        page = None
 
-        # Accepter les cookies si nécessaire
-        await _accept_cookies(page, domain)
+        try:
+            # Get random User-Agent for each attempt (important for Amazon)
+            current_user_agent = _get_random_user_agent()
 
-        # Attendre que les résultats se chargent
-        if wait_selector:
-            try:
-                await page.wait_for_selector(wait_selector, timeout=10000)
-            except Exception:
-                logger.debug(f"Selector {wait_selector} non trouvé sur {domain}, on continue...")
+            # Create stealth headers - more comprehensive for Amazon
+            extra_headers = {
+                "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Cache-Control": "max-age=0",
+                "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+                "Sec-Ch-Ua-Mobile": "?0",
+                "Sec-Ch-Ua-Platform": '"Windows"',
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+            }
 
-        # Attendre un peu pour le JS (réduit pour parallélisation)
-        await asyncio.sleep(1.5)
+            # Add referer for Amazon to look more natural
+            if is_amazon:
+                extra_headers["Referer"] = "https://www.google.fr/"
 
-        # Extraire le HTML
-        html_content = await page.content()
+            # Créer un nouveau contexte isolé pour ce site
+            context = await browser.new_context(
+                viewport={"width": 1920, "height": 1080},
+                user_agent=current_user_agent,
+                locale="fr-FR",
+                timezone_id="Europe/Paris",
+                extra_http_headers=extra_headers,
+            )
 
-        # Check for CAPTCHA
-        if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
-            logger.warning(f"CAPTCHA detected on {domain}!")
-            _dump_debug_html(html_content, domain, query)
-            return []
+            # For Amazon, don't block images - it may trigger detection
+            if not is_amazon:
+                await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+            await context.route("**/analytics*", lambda route: route.abort())
+            await context.route("**/tracking*", lambda route: route.abort())
 
-        # Parser le HTML avec BeautifulSoup
-        soup = BeautifulSoup(html_content, "lxml")
-        results = []
-        seen_urls = set()
+            page = await context.new_page()
 
-        # Trouver les liens produits
-        links = []
-        if product_selector:
-            # Essayer plusieurs sélecteurs séparés par virgule
-            selectors = [s.strip() for s in product_selector.split(",")]
-            for sel in selectors:
+            # Add random delay before navigation for Amazon
+            if is_amazon:
+                await _random_delay(1.0, 3.0)
+
+            # Naviguer vers la page de recherche
+            await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
+
+            # For Amazon, simulate human behavior before interacting
+            if is_amazon:
+                await _simulate_human_behavior(page)
+
+            # Accepter les cookies si nécessaire
+            await _accept_cookies(page, domain)
+
+            # Attendre que les résultats se chargent
+            if wait_selector:
                 try:
-                    found = soup.select(sel)
-                    links.extend(found)
+                    await page.wait_for_selector(wait_selector, timeout=10000)
                 except Exception:
+                    logger.debug(f"Selector {wait_selector} non trouvé sur {domain}, on continue...")
+
+            # Attendre un peu pour le JS - longer wait for Amazon
+            wait_time = random.uniform(2.0, 4.0) if is_amazon else 1.5
+            await asyncio.sleep(wait_time)
+
+            # Extraire le HTML
+            html_content = await page.content()
+
+            # Check for Amazon blocking (503, "Toutes nos excuses", etc.)
+            if is_amazon and _is_amazon_blocked(html_content):
+                logger.warning(f"Amazon blocked request (attempt {attempt + 1}/{max_retries}) for '{query}'")
+                _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
+                # Close resources before retry
+                try:
+                    if page:
+                        await page.close()
+                    if context:
+                        await context.close()
+                except Exception:
+                    pass
+                continue  # Retry with new User-Agent and delay
+
+            # Check for CAPTCHA
+            if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
+                logger.warning(f"CAPTCHA detected on {domain}!")
+                _dump_debug_html(html_content, domain, query)
+                if is_amazon and attempt < max_retries - 1:
+                    try:
+                        if page:
+                            await page.close()
+                        if context:
+                            await context.close()
+                    except Exception:
+                        pass
+                    continue  # Retry
+                return []
+
+            # Parser le HTML avec BeautifulSoup
+            soup = BeautifulSoup(html_content, "lxml")
+            results = []
+            seen_urls = set()
+
+            # Trouver les liens produits
+            links = []
+            if product_selector:
+                # Essayer plusieurs sélecteurs séparés par virgule
+                selectors = [s.strip() for s in product_selector.split(",")]
+                for sel in selectors:
+                    try:
+                        found = soup.select(sel)
+                        links.extend(found)
+                    except Exception:
+                        continue
+                logger.info(f"{domain}: {len(links)} liens trouvés avec sélecteur configuré")
+
+            # Fallback: détection générique si le sélecteur n'a rien trouvé
+            if not links:
+                logger.info(f"{domain}: Sélecteur configuré n'a rien trouvé, utilisation détection générique")
+                links = _find_product_links(soup, domain)
+                logger.info(f"{domain}: {len(links)} liens trouvés avec détection générique")
+
+            for link in links:
+                href = link.get("href", "")
+                if not href:
                     continue
-            logger.info(f"{domain}: {len(links)} liens trouvés avec sélecteur configuré")
 
-        # Fallback: détection générique si le sélecteur n'a rien trouvé
-        if not links:
-            logger.info(f"{domain}: Sélecteur configuré n'a rien trouvé, utilisation détection générique")
-            links = _find_product_links(soup, domain)
-            logger.info(f"{domain}: {len(links)} liens trouvés avec détection générique")
+                # Construire l'URL complète
+                if href.startswith("/"):
+                    href = f"https://{domain}{href}"
+                elif not href.startswith("http"):
+                    href = urljoin(final_url, href)
 
-        for link in links:
-            href = link.get("href", "")
-            if not href:
-                continue
+                # Nettoyer l'URL
+                href = _clean_product_url(href, domain)
 
-            # Construire l'URL complète
-            if href.startswith("/"):
-                href = f"https://{domain}{href}"
-            elif not href.startswith("http"):
-                href = urljoin(final_url, href)
+                # Éviter les doublons
+                if href in seen_urls:
+                    continue
+                seen_urls.add(href)
 
-            # Nettoyer l'URL
-            href = _clean_product_url(href, domain)
+                # Vérifier que c'est bien un lien vers ce domaine
+                url_domain = _extract_domain(href)
+                if domain not in url_domain and url_domain not in domain:
+                    # logger.debug(f"Ignored external link: {href}")
+                    continue
 
-            # Éviter les doublons
-            if href in seen_urls:
-                continue
-            seen_urls.add(href)
+                # Filtrer les liens non-produits
+                if _is_non_product_url(href):
+                    # logger.debug(f"Ignored non-product link: {href}")
+                    continue
 
-            # Vérifier que c'est bien un lien vers ce domaine
-            url_domain = _extract_domain(href)
-            if domain not in url_domain and url_domain not in domain:
-                # logger.debug(f"Ignored external link: {href}")
-                continue
+                # Extraire le titre
+                title = _extract_title(link)
+                if not title or len(title) < MIN_TITLE_LENGTH:
+                    # logger.debug(f"Ignored link with no title: {href}")
+                    continue
 
-            # Filtrer les liens non-produits
-            if _is_non_product_url(href):
-                # logger.debug(f"Ignored non-product link: {href}")
-                continue
+                results.append(SearchResult(
+                    url=href,
+                    title=title,
+                    snippet="",
+                    source=domain,
+                    # On ne peut pas extraire le prix facilement ici sans scraper chaque page
+                    # Le scraping détaillé se fera dans la phase 2
+                ))
 
-            # Extraire le titre
-            title = _extract_title(link)
-            if not title or len(title) < MIN_TITLE_LENGTH:
-                # logger.debug(f"Ignored link with no title: {href}")
-                continue
+                if len(results) >= max_results:
+                    break
 
-            results.append(SearchResult(
-                url=href,
-                title=title,
-                snippet="",
-                source=domain,
-                # On ne peut pas extraire le prix facilement ici sans scraper chaque page
-                # Le scraping détaillé se fera dans la phase 2
-            ))
+            logger.info(f"{domain}: {len(results)} résultats extraits")
 
-            if len(results) >= max_results:
-                break
+            # If we got results, success! Return them
+            if len(results) > 0:
+                return results
 
-        logger.info(f"{domain}: {len(results)} résultats extraits")
-        
-        if len(results) == 0:
+            # No results found - for Amazon this might be temporary blocking
             logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
             _dump_debug_html(html_content, domain, query)
-            
-        return results
 
-    except Exception as e:
-        logger.error(f"Erreur Browserless sur {domain}: {e}")
-        return []
-        
-    finally:
-        # Fermer le contexte et la page, mais PAS le navigateur
-        try:
-            if page:
-                await page.close()
-            if context:
-                await context.close()
+            # For Amazon, if no results and we have retries left, try again
+            if is_amazon and attempt < max_retries - 1:
+                logger.info(f"Amazon returned 0 results, will retry...")
+                continue
+
+            return results
+
         except Exception as e:
-            logger.debug(f"Erreur fermeture contexte {domain}: {e}")
+            logger.error(f"Erreur Browserless sur {domain}: {e}")
+            if is_amazon and attempt < max_retries - 1:
+                logger.info(f"Will retry after error...")
+                continue
+            return []
+
+        finally:
+            # Fermer le contexte et la page, mais PAS le navigateur
+            try:
+                if page:
+                    await page.close()
+                if context:
+                    await context.close()
+            except Exception as e:
+                logger.debug(f"Erreur fermeture contexte {domain}: {e}")
+
+    # If we exhausted all retries for Amazon
+    logger.error(f"All {max_retries} attempts failed for Amazon search '{query}'")
+    return []
 
 
 def _get_default_search_url(domain: str) -> str | None:
diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py
index ffa7b24..be0c931 100644
--- a/app/services/scraper_service.py
+++ b/app/services/scraper_service.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import random
 import re
 from urllib.parse import urlparse, urlunparse
 
@@ -15,6 +16,50 @@ BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 MAX_RETRIES = 2
 RETRY_DELAY = 3  # secondes
 
+# Amazon-specific configuration
+AMAZON_MAX_RETRIES = 4
+AMAZON_BASE_DELAY = 3.0
+
+# Pool of realistic User-Agents for rotation
+USER_AGENT_POOL = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+]
+
+# Amazon blocking indicators
+AMAZON_BLOCK_INDICATORS = [
+    "Toutes nos excuses",
+    "Sorry, something went wrong",
+    "api-services-support@amazon.com",
+    "Enter the characters you see below",
+    "Saisissez les caractères",
+    "Service Unavailable",
+    "robot check",
+    "automated access",
+]
+
+
+def _get_random_user_agent() -> str:
+    """Get a random User-Agent from the pool"""
+    return random.choice(USER_AGENT_POOL)
+
+
+def _is_amazon_url(url: str) -> bool:
+    """Check if URL is an Amazon URL"""
+    return "amazon" in url.lower()
+
+
+def _is_amazon_blocked(html_content: str) -> bool:
+    """Check if Amazon has blocked the request"""
+    html_lower = html_content.lower()
+    for indicator in AMAZON_BLOCK_INDICATORS:
+        if indicator.lower() in html_lower:
+            return True
+    return False
+
 
 def simplify_url(url: str) -> str:
     """
@@ -117,12 +162,23 @@ class ScraperService:
             logger.warning(f"Invalid timeout value: {timeout}, using default 90000")
             timeout = 90000
 
+        # Amazon needs more retries with longer delays
+        is_amazon = _is_amazon_url(url)
+        max_retries = AMAZON_MAX_RETRIES if is_amazon else MAX_RETRIES
+        base_delay = AMAZON_BASE_DELAY if is_amazon else RETRY_DELAY
+
         # Retries avec backoff
         last_error = None
-        for attempt in range(MAX_RETRIES + 1):
+        for attempt in range(max_retries + 1):
             if attempt > 0:
-                logger.info(f"Tentative {attempt + 1}/{MAX_RETRIES + 1} pour {url}")
-                await asyncio.sleep(RETRY_DELAY * attempt)
+                # Exponential backoff with jitter for Amazon
+                if is_amazon:
+                    jitter = random.uniform(0.5, 1.5)
+                    delay = min(base_delay * (2 ** attempt) * jitter, 30.0)
+                else:
+                    delay = base_delay * attempt
+                logger.info(f"Tentative {attempt + 1}/{max_retries + 1} pour {url} (délai: {delay:.1f}s)")
+                await asyncio.sleep(delay)
 
             result = await ScraperService._do_scrape(
                 url=url,
@@ -133,6 +189,7 @@ class ScraperService:
                 text_length=text_length,
                 timeout=timeout,
                 browser=browser,
+                is_amazon=is_amazon,
             )
 
             if result[0] is not None:  # Screenshot réussi
@@ -154,14 +211,15 @@ class ScraperService:
         text_length: int = 0,
         timeout: int = 90000,
         browser=None,
+        is_amazon: bool = False,
     ) -> tuple[str | None, str, bool]:
         """Exécute le scraping réel (appelé par scrape_item avec retries)."""
-        
+
         # Si un navigateur est fourni, on l'utilise directement sans async_playwright context manager
         # sinon on crée tout de zéro
         playwright_manager = None
         local_browser = None
-        
+
         try:
             if browser:
                 # Utiliser le navigateur partagé
@@ -180,22 +238,37 @@ class ScraperService:
             context = None
             page = None
             is_available = True  # Par défaut, le produit est disponible
-            
+
+            # Use random User-Agent for anti-detection (especially for Amazon)
+            current_user_agent = _get_random_user_agent()
+
+            # Enhanced headers for stealth
+            extra_headers = {
+                "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Cache-Control": "max-age=0",
+                "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+                "Sec-Ch-Ua-Mobile": "?0",
+                "Sec-Ch-Ua-Platform": '"Windows"',
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+
+            # Add referer for Amazon
+            if is_amazon:
+                extra_headers["Referer"] = "https://www.google.fr/"
+
             try:
                 context = await current_browser.new_context(
                     viewport={"width": 1920, "height": 1080},
-                    user_agent=(
-                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                        "AppleWebKit/537.36 (KHTML, like Gecko) "
-                        "Chrome/120.0.0.0 Safari/537.36"
-                    ),
+                    user_agent=current_user_agent,
                     locale="fr-FR",
                     timezone_id="Europe/Paris",
-                    # Options supplémentaires pour éviter la détection
-                    extra_http_headers={
-                        "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
-                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-                    }
+                    extra_http_headers=extra_headers,
                 )
 
                 # Stealth mode / Ad blocking attempts
@@ -203,25 +276,37 @@ class ScraperService:
 
                 page = await context.new_page()
 
+                # Add random delay before navigation for Amazon
+                if is_amazon:
+                    await asyncio.sleep(random.uniform(1.0, 3.0))
+
                 logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
                 response = None
+                amazon_blocked = False
                 try:
                     # First wait for domcontentloaded - this is the minimum we need
                     response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
-                    
+
                     # Check HTTP status code
                     if response:
                         status = response.status
                         logger.info(f"HTTP Status: {status}")
-                        
-                        # Detect unavailable products by HTTP status
-                        if status in [404, 410, 451, 503]:
+
+                        # For Amazon, 503 is usually a block, not product unavailability
+                        if is_amazon and status == 503:
+                            logger.warning(f"Amazon returned 503 - likely blocking")
+                            amazon_blocked = True
+                        # Detect unavailable products by HTTP status (non-Amazon)
+                        elif status in [404, 410, 451]:
+                            logger.warning(f"Product unavailable - HTTP {status}")
+                            is_available = False
+                        elif status == 503 and not is_amazon:
                             logger.warning(f"Product unavailable - HTTP {status}")
                             is_available = False
                         elif status >= 400:
                             logger.warning(f"HTTP error {status}, marking as potentially unavailable")
                             is_available = False
-                    
+
                     logger.info(f"Page loaded (domcontentloaded): {url}")
 
                     # Then try to wait for networkidle, but don't fail if it times out
@@ -239,8 +324,24 @@ class ScraperService:
                     # Try to take screenshot anyway if page partially loaded
                     pass
 
-                # Wait a bit for dynamic content if needed
-                await page.wait_for_timeout(2000)
+                # Wait a bit for dynamic content if needed - longer for Amazon
+                wait_time = random.uniform(2.0, 4.0) if is_amazon else 2.0
+                await page.wait_for_timeout(int(wait_time * 1000))
+
+                # Check for Amazon blocking in page content
+                if is_amazon:
+                    try:
+                        page_html = await page.content()
+                        if _is_amazon_blocked(page_html):
+                            logger.warning(f"Amazon blocking detected in page content for {url}")
+                            amazon_blocked = True
+                    except Exception as e:
+                        logger.debug(f"Could not check Amazon blocking: {e}")
+
+                # If Amazon blocked us, return None to trigger retry
+                if amazon_blocked:
+                    logger.warning(f"Amazon blocked - will retry with different User-Agent")
+                    return None, "", True  # Return None screenshot to trigger retry
 
                 # Try to close common popups and cookie banners
                 logger.info("Attempting to close popups and cookie banners...")

From e62117f9b9bff1eb3d64a2d2b649ef288c629878 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:01:39 +0000
Subject: [PATCH 04/13] feat: Add per-site debug toggle for HTML dump control

- Add debug_enabled field to SearchSite model
- Update schemas and API routes to allow debug toggle
- Add debug toggle button on site cards in Admin UI
- Only dump HTML when debug is enabled for the site (except for critical errors like Amazon blocking)
- Add API endpoints to list/delete debug files per site domain
- Add button to clear debug files for individual sites
---
 app/models.py                         |  1 +
 app/routers/debug.py                  | 71 +++++++++++++++++++++++--
 app/routers/search_sites.py           |  6 +--
 app/schemas.py                        |  3 ++
 app/services/direct_search_service.py | 18 +++++--
 app/services/search_service.py        |  1 +
 frontend/src/pages/Admin.jsx          | 74 ++++++++++++++++++++++++---
 7 files changed, 157 insertions(+), 17 deletions(-)

diff --git a/app/models.py b/app/models.py
index cbee95e..d5d61e3 100644
--- a/app/models.py
+++ b/app/models.py
@@ -94,6 +94,7 @@ class SearchSite(Base):
     is_active: bool = Column(Boolean, default=True)  # type: ignore
     priority: int = Column(Integer, default=0)  # type: ignore  # Ordre d'affichage
     requires_js: bool = Column(Boolean, default=False)  # type: ignore  # Force Browserless si True
+    debug_enabled: bool = Column(Boolean, default=False)  # type: ignore  # Activer le dump HTML pour ce site
     price_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour le prix
     search_url: str | None = Column(String, nullable=True)  # type: ignore  # URL de recherche avec {query} placeholder
     product_link_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour les liens produits
diff --git a/app/routers/debug.py b/app/routers/debug.py
index 54a9b25..4714a02 100644
--- a/app/routers/debug.py
+++ b/app/routers/debug.py
@@ -75,19 +75,82 @@ def delete_debug_dump(filename: str):
     """Delete a specific debug dump file"""
     if not os.path.exists(DEBUG_DIR):
         raise HTTPException(status_code=404, detail="Debug directory not found")
-    
+
     filepath = os.path.join(DEBUG_DIR, filename)
-    
+
     # Security check: prevent directory traversal
     if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)):
         raise HTTPException(status_code=403, detail="Access denied")
-        
+
     if not os.path.isfile(filepath):
         raise HTTPException(status_code=404, detail="File not found")
-    
+
     try:
         os.remove(filepath)
         return {"message": "File deleted"}
     except Exception as e:
         logger.error(f"Error deleting debug dump {filename}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/dumps/site/{domain}", response_model=List[DebugFile])
+def list_debug_dumps_by_site(domain: str):
+    """List debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return []
+
+    # Clean domain for comparison (remove www. if present)
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    files = []
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            # Check if filename starts with the domain
+            filename_lower = filename.lower()
+            # Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    stat = os.stat(filepath)
+                    files.append(DebugFile(
+                        filename=filename,
+                        size=stat.st_size,
+                        created_at=stat.st_ctime,
+                        created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S")
+                    ))
+
+        # Sort by creation time (newest first)
+        files.sort(key=lambda x: x.created_at, reverse=True)
+        return files
+    except Exception as e:
+        logger.error(f"Error listing debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/dumps/site/{domain}")
+def delete_debug_dumps_by_site(domain: str):
+    """Delete all debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return {"message": "No files found", "deleted": 0}
+
+    # Clean domain for comparison
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    deleted_count = 0
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            filename_lower = filename.lower()
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    os.remove(filepath)
+                    deleted_count += 1
+
+        return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count}
+    except Exception as e:
+        logger.error(f"Error deleting debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
diff --git a/app/routers/search_sites.py b/app/routers/search_sites.py
index 9ae6bff..5df10bc 100644
--- a/app/routers/search_sites.py
+++ b/app/routers/search_sites.py
@@ -41,17 +41,17 @@ async def update_site(
 ):
     """
     Met à jour un site de recherche.
-    Permet uniquement de modifier is_active et priority.
+    Permet uniquement de modifier is_active, priority et debug_enabled.
     """
     # Limiter les champs modifiables
-    allowed_fields = {"is_active", "priority"}
+    allowed_fields = {"is_active", "priority", "debug_enabled"}
     update_data = site.model_dump(exclude_unset=True)
     filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields}
 
     if not filtered_data:
         raise HTTPException(
             status_code=400,
-            detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés"
+            detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés"
         )
 
     updated = search_service.update_site(db, site_id, filtered_data)
diff --git a/app/schemas.py b/app/schemas.py
index 2f82e75..e34886c 100644
--- a/app/schemas.py
+++ b/app/schemas.py
@@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel):
     is_active: bool = True
     priority: int = 0
     requires_js: bool = False
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None  # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query}
     product_link_selector: str | None = None  # Sélecteur CSS pour les liens produits
@@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel):
     is_active: bool | None = None
     priority: int | None = None
     requires_js: bool | None = None
+    debug_enabled: bool | None = None  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None
@@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel):
     is_active: bool
     priority: int
     requires_js: bool
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None
diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 995c39f..f937bc8 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -631,6 +631,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
     # Nettoyer le domaine (enlever protocole, www, slash final)
     domain = _clean_domain(raw_domain)
 
+    # Check if debug is enabled for this site
+    debug_enabled = site.get("debug_enabled", False)
+
     search_url = site.get("search_url") or _get_default_search_url(domain)
     product_selector = site.get("product_link_selector") or _get_default_product_selector(domain)
     wait_selector = _get_default_wait_selector(domain)
@@ -733,7 +736,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             # Check for Amazon blocking (503, "Toutes nos excuses", etc.)
             if is_amazon and _is_amazon_blocked(html_content):
                 logger.warning(f"Amazon blocked request (attempt {attempt + 1}/{max_retries}) for '{query}'")
-                _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
+                # Always dump for Amazon blocking (critical error)
+                if debug_enabled or is_amazon:
+                    _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
                 # Close resources before retry
                 try:
                     if page:
@@ -747,7 +752,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             # Check for CAPTCHA
             if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
                 logger.warning(f"CAPTCHA detected on {domain}!")
-                _dump_debug_html(html_content, domain, query)
+                # Always dump for CAPTCHA (critical error)
+                if debug_enabled or is_amazon:
+                    _dump_debug_html(html_content, domain, f"{query}_captcha")
                 if is_amazon and attempt < max_retries - 1:
                     try:
                         if page:
@@ -838,8 +845,11 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 return results
 
             # No results found - for Amazon this might be temporary blocking
-            logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
-            _dump_debug_html(html_content, domain, query)
+            if debug_enabled:
+                logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
+                _dump_debug_html(html_content, domain, f"{query}_no_results")
+            else:
+                logger.warning(f"{domain}: 0 results found (debug disabled, no HTML dump)")
 
             # For Amazon, if no results and we have retries left, try again
             if is_amazon and attempt < max_retries - 1:
diff --git a/app/services/search_service.py b/app/services/search_service.py
index 4b6f66b..249ba72 100644
--- a/app/services/search_service.py
+++ b/app/services/search_service.py
@@ -76,6 +76,7 @@ async def search_products(
             "search_url": site.search_url,
             "product_link_selector": site.product_link_selector,
             "requires_js": site.requires_js,
+            "debug_enabled": site.debug_enabled,
         }
         for site in sites
     ]
diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx
index abb631b..d08fcba 100644
--- a/frontend/src/pages/Admin.jsx
+++ b/frontend/src/pages/Admin.jsx
@@ -437,13 +437,51 @@ export default function Admin() {
     // Site functions
     const toggleSiteActive = async (site) => {
         try {
-            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active });
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() });
             fetchAll();
         } catch (error) {
             toast.error('Erreur lors de la mise à jour');
         }
     };
 
+    const toggleSiteDebug = async (site) => {
+        try {
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() });
+            toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé');
+            fetchAll();
+        } catch (error) {
+            toast.error('Erreur lors de la mise à jour du debug');
+        }
+    };
+
+    const fetchSiteDebugFiles = async (domain) => {
+        try {
+            const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            return response.data;
+        } catch (error) {
+            console.error('Error fetching debug files:', error);
+            return [];
+        }
+    };
+
+    const downloadDebugFile = async (filename) => {
+        try {
+            window.open(`${API_URL}/debug/dumps/${filename}`, '_blank');
+        } catch (error) {
+            toast.error('Erreur lors du téléchargement');
+        }
+    };
+
+    const deleteSiteDebugFiles = async (domain) => {
+        if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return;
+        try {
+            const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            toast.success(response.data.message);
+        } catch (error) {
+            toast.error('Erreur lors de la suppression');
+        }
+    };
+
     const resetSitesToDefaults = async () => {
         if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return;
         setResettingSites(true);
@@ -862,11 +900,35 @@ export default function Admin() {
                                         </div>
                                         <Switch checked={site.is_active} onCheckedChange={() => toggleSiteActive(site)} className="ml-2" />
                                     </div>
-                                    {site.category && (
-                                        <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
-                                            {site.category}
-                                        </span>
-                                    )}
+                                    <div className="flex items-center justify-between">
+                                        {site.category && (
+                                            <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
+                                                {site.category}
+                                            </span>
+                                        )}
+                                        <div className="flex items-center gap-2 ml-auto">
+                                            <Button
+                                                variant="ghost"
+                                                size="sm"
+                                                className={cn("h-7 px-2", site.debug_enabled && "text-orange-500")}
+                                                onClick={() => toggleSiteDebug(site)}
+                                                title={site.debug_enabled ? "Désactiver le debug" : "Activer le debug"}
+                                            >
+                                                <Bug className="h-4 w-4" />
+                                            </Button>
+                                            {site.debug_enabled && (
+                                                <Button
+                                                    variant="ghost"
+                                                    size="sm"
+                                                    className="h-7 px-2"
+                                                    onClick={() => deleteSiteDebugFiles(site.domain)}
+                                                    title="Supprimer fichiers debug"
+                                                >
+                                                    <Trash2 className="h-4 w-4" />
+                                                </Button>
+                                            )}
+                                        </div>
+                                    </div>
                                 </div>
                             ))}
                         </div>

From 3bae422c38e935453f2217cd27c1202580d515cc Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:07:49 +0000
Subject: [PATCH 05/13] fix: Add stealth JavaScript injection to bypass Amazon
 bot detection

- Add STEALTH_JS constant with scripts to hide automation fingerprints
- Override navigator.webdriver, plugins, languages, permissions
- Fake WebGL vendor/renderer and hardware concurrency
- Inject stealth script via context.add_init_script() for Amazon pages
- Add bypass_csp=True to allow script injection
- Don't block images for Amazon to avoid detection triggers
- Apply stealth to both direct_search_service and scraper_service
---
 app/services/direct_search_service.py | 108 ++++++++++++++++++++++++
 app/services/scraper_service.py       | 116 +++++++++++++++++++++++++-
 2 files changed, 222 insertions(+), 2 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index f937bc8..5b0a350 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -109,6 +109,106 @@ async def _simulate_human_behavior(page):
 USER_AGENT = USER_AGENT_POOL[0]
 
 
+# Stealth JavaScript to inject before page load to hide automation
+STEALTH_JS = """
+// Override the navigator.webdriver property
+Object.defineProperty(navigator, 'webdriver', {
+    get: () => undefined,
+    configurable: true
+});
+
+// Override navigator.plugins to have some plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+        ];
+        plugins.item = (i) => plugins[i];
+        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.refresh = () => {};
+        return plugins;
+    },
+    configurable: true
+});
+
+// Override navigator.languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['fr-FR', 'fr', 'en-US', 'en'],
+    configurable: true
+});
+
+// Override permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) => (
+    parameters.name === 'notifications' ?
+        Promise.resolve({ state: Notification.permission }) :
+        originalQuery(parameters)
+);
+
+// Hide automation-related properties
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+// Override chrome object
+window.chrome = {
+    runtime: {},
+    loadTimes: function() {},
+    csi: function() {},
+    app: {}
+};
+
+// Fake WebGL vendor and renderer
+const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+        if (args[0] === 37445) {
+            return 'Intel Inc.';
+        }
+        if (args[0] === 37446) {
+            return 'Intel Iris OpenGL Engine';
+        }
+        return Reflect.apply(target, ctx, args);
+    }
+};
+
+try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+        const getParameter = gl.getParameter.bind(gl);
+        gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
+    }
+} catch(e) {}
+
+// Override connection rtt (reduces fingerprinting)
+Object.defineProperty(navigator, 'connection', {
+    get: () => ({
+        effectiveType: '4g',
+        rtt: 50,
+        downlink: 10,
+        saveData: false
+    }),
+    configurable: true
+});
+
+// Override hardware concurrency
+Object.defineProperty(navigator, 'hardwareConcurrency', {
+    get: () => 8,
+    configurable: true
+});
+
+// Override device memory
+Object.defineProperty(navigator, 'deviceMemory', {
+    get: () => 8,
+    configurable: true
+});
+
+console.log('Stealth mode activated');
+"""
+
+
 # Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py)
 DEBUG_DUMPS_DIR = "/app/debug_dumps"
 
@@ -695,8 +795,16 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 locale="fr-FR",
                 timezone_id="Europe/Paris",
                 extra_http_headers=extra_headers,
+                # Additional stealth options
+                java_script_enabled=True,
+                bypass_csp=True,  # Bypass Content Security Policy for script injection
             )
 
+            # Inject stealth script before any page loads (especially for Amazon)
+            if is_amazon:
+                await context.add_init_script(STEALTH_JS)
+                logger.debug("Stealth JS injected for Amazon")
+
             # For Amazon, don't block images - it may trigger detection
             if not is_amazon:
                 await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py
index be0c931..e0e1d18 100644
--- a/app/services/scraper_service.py
+++ b/app/services/scraper_service.py
@@ -41,6 +41,105 @@ AMAZON_BLOCK_INDICATORS = [
     "automated access",
 ]
 
+# Stealth JavaScript to inject before page load to hide automation
+STEALTH_JS = """
+// Override the navigator.webdriver property
+Object.defineProperty(navigator, 'webdriver', {
+    get: () => undefined,
+    configurable: true
+});
+
+// Override navigator.plugins to have some plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+        ];
+        plugins.item = (i) => plugins[i];
+        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.refresh = () => {};
+        return plugins;
+    },
+    configurable: true
+});
+
+// Override navigator.languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['fr-FR', 'fr', 'en-US', 'en'],
+    configurable: true
+});
+
+// Override permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) => (
+    parameters.name === 'notifications' ?
+        Promise.resolve({ state: Notification.permission }) :
+        originalQuery(parameters)
+);
+
+// Hide automation-related properties
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+// Override chrome object
+window.chrome = {
+    runtime: {},
+    loadTimes: function() {},
+    csi: function() {},
+    app: {}
+};
+
+// Fake WebGL vendor and renderer
+const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+        if (args[0] === 37445) {
+            return 'Intel Inc.';
+        }
+        if (args[0] === 37446) {
+            return 'Intel Iris OpenGL Engine';
+        }
+        return Reflect.apply(target, ctx, args);
+    }
+};
+
+try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+        const getParameter = gl.getParameter.bind(gl);
+        gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
+    }
+} catch(e) {}
+
+// Override connection rtt (reduces fingerprinting)
+Object.defineProperty(navigator, 'connection', {
+    get: () => ({
+        effectiveType: '4g',
+        rtt: 50,
+        downlink: 10,
+        saveData: false
+    }),
+    configurable: true
+});
+
+// Override hardware concurrency
+Object.defineProperty(navigator, 'hardwareConcurrency', {
+    get: () => 8,
+    configurable: true
+});
+
+// Override device memory
+Object.defineProperty(navigator, 'deviceMemory', {
+    get: () => 8,
+    configurable: true
+});
+
+console.log('Stealth mode activated');
+"""
+
 
 def _get_random_user_agent() -> str:
     """Get a random User-Agent from the pool"""
@@ -269,10 +368,23 @@ class ScraperService:
                     locale="fr-FR",
                     timezone_id="Europe/Paris",
                     extra_http_headers=extra_headers,
+                    # Additional stealth options
+                    java_script_enabled=True,
+                    bypass_csp=True,  # Bypass Content Security Policy for script injection
                 )
 
-                # Stealth mode / Ad blocking attempts
-                await context.route("**/*", lambda route: route.continue_())
+                # Inject stealth script before any page loads (especially for Amazon)
+                if is_amazon:
+                    await context.add_init_script(STEALTH_JS)
+                    logger.debug("Stealth JS injected for Amazon product page")
+
+                # Stealth mode / Ad blocking attempts - don't block images for Amazon
+                if is_amazon:
+                    # For Amazon, only block tracking - keep images to avoid detection
+                    await context.route("**/analytics*", lambda route: route.abort())
+                    await context.route("**/tracking*", lambda route: route.abort())
+                else:
+                    await context.route("**/*", lambda route: route.continue_())
 
                 page = await context.new_page()
 

From 8a9ec6a6db46b58796050670ab7b0151bea9a845 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:11:15 +0000
Subject: [PATCH 06/13] fix: Add database migration for search_sites missing
 columns

Add migration to add debug_enabled, price_selector, search_url, and
product_link_selector columns to the search_sites table.
---
 .../versions/add_search_sites_columns.py      | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 app/alembic/versions/add_search_sites_columns.py

diff --git a/app/alembic/versions/add_search_sites_columns.py b/app/alembic/versions/add_search_sites_columns.py
new file mode 100644
index 0000000..703755e
--- /dev/null
+++ b/app/alembic/versions/add_search_sites_columns.py
@@ -0,0 +1,49 @@
+"""Add missing columns to search_sites table
+
+Revision ID: add_search_sites_columns
+Revises: merge_heads
+Create Date: 2025-11-24
+
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "add_search_sites_columns"
+down_revision: str | None = "merge_heads"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add debug_enabled, price_selector, search_url, product_link_selector columns to search_sites."""
+    # Add debug_enabled column
+    op.add_column(
+        "search_sites",
+        sa.Column("debug_enabled", sa.Boolean(), server_default="false", nullable=False),
+    )
+    # Add price_selector column
+    op.add_column(
+        "search_sites",
+        sa.Column("price_selector", sa.String(), nullable=True),
+    )
+    # Add search_url column
+    op.add_column(
+        "search_sites",
+        sa.Column("search_url", sa.String(), nullable=True),
+    )
+    # Add product_link_selector column
+    op.add_column(
+        "search_sites",
+        sa.Column("product_link_selector", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    """Remove added columns from search_sites."""
+    op.drop_column("search_sites", "product_link_selector")
+    op.drop_column("search_sites", "search_url")
+    op.drop_column("search_sites", "price_selector")
+    op.drop_column("search_sites", "debug_enabled")

From 898e19cf0a586b30cb013850814d6e50c440d759 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:12:05 +0000
Subject: [PATCH 07/13] fix: Remove duplicate downloadDebugFile function
 causing build failure

---
 frontend/src/pages/Admin.jsx | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx
index d08fcba..1d7af6f 100644
--- a/frontend/src/pages/Admin.jsx
+++ b/frontend/src/pages/Admin.jsx
@@ -464,14 +464,6 @@ export default function Admin() {
         }
     };
 
-    const downloadDebugFile = async (filename) => {
-        try {
-            window.open(`${API_URL}/debug/dumps/${filename}`, '_blank');
-        } catch (error) {
-            toast.error('Erreur lors du téléchargement');
-        }
-    };
-
     const deleteSiteDebugFiles = async (domain) => {
         if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return;
         try {

From 30861acdb8d2bcc0fff93e162a0ea2310e3d9a87 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:14:39 +0000
Subject: [PATCH 08/13] fix: Correct migration dependency chain to avoid
 multiple heads

---
 app/alembic/versions/add_search_sites_columns.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/alembic/versions/add_search_sites_columns.py b/app/alembic/versions/add_search_sites_columns.py
index 703755e..b0def85 100644
--- a/app/alembic/versions/add_search_sites_columns.py
+++ b/app/alembic/versions/add_search_sites_columns.py
@@ -12,7 +12,7 @@ from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "add_search_sites_columns"
-down_revision: str | None = "merge_heads"
+down_revision: str | None = "add_notification_channels"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

From 2384fca9ca22a6a51a9fa649634b121fa6e7a7d3 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:17:49 +0000
Subject: [PATCH 09/13] fix: Make migration idempotent - check if columns exist
 before adding

---
 .../versions/add_search_sites_columns.py      | 70 ++++++++++++-------
 1 file changed, 45 insertions(+), 25 deletions(-)

diff --git a/app/alembic/versions/add_search_sites_columns.py b/app/alembic/versions/add_search_sites_columns.py
index b0def85..b6d49be 100644
--- a/app/alembic/versions/add_search_sites_columns.py
+++ b/app/alembic/versions/add_search_sites_columns.py
@@ -1,7 +1,7 @@
 """Add missing columns to search_sites table
 
 Revision ID: add_search_sites_columns
-Revises: merge_heads
+Revises: add_notification_channels
 Create Date: 2025-11-24
 
 """
@@ -9,6 +9,7 @@ from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
+from sqlalchemy import inspect
 
 # revision identifiers, used by Alembic.
 revision: str = "add_search_sites_columns"
@@ -17,33 +18,52 @@ branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 
 
+def column_exists(table_name: str, column_name: str) -> bool:
+    """Check if a column exists in a table."""
+    bind = op.get_bind()
+    inspector = inspect(bind)
+    columns = [col["name"] for col in inspector.get_columns(table_name)]
+    return column_name in columns
+
+
 def upgrade() -> None:
     """Add debug_enabled, price_selector, search_url, product_link_selector columns to search_sites."""
-    # Add debug_enabled column
-    op.add_column(
-        "search_sites",
-        sa.Column("debug_enabled", sa.Boolean(), server_default="false", nullable=False),
-    )
-    # Add price_selector column
-    op.add_column(
-        "search_sites",
-        sa.Column("price_selector", sa.String(), nullable=True),
-    )
-    # Add search_url column
-    op.add_column(
-        "search_sites",
-        sa.Column("search_url", sa.String(), nullable=True),
-    )
-    # Add product_link_selector column
-    op.add_column(
-        "search_sites",
-        sa.Column("product_link_selector", sa.String(), nullable=True),
-    )
+    # Add debug_enabled column (only if not exists)
+    if not column_exists("search_sites", "debug_enabled"):
+        op.add_column(
+            "search_sites",
+            sa.Column("debug_enabled", sa.Boolean(), server_default="false", nullable=False),
+        )
+
+    # Add price_selector column (only if not exists)
+    if not column_exists("search_sites", "price_selector"):
+        op.add_column(
+            "search_sites",
+            sa.Column("price_selector", sa.String(), nullable=True),
+        )
+
+    # Add search_url column (only if not exists)
+    if not column_exists("search_sites", "search_url"):
+        op.add_column(
+            "search_sites",
+            sa.Column("search_url", sa.String(), nullable=True),
+        )
+
+    # Add product_link_selector column (only if not exists)
+    if not column_exists("search_sites", "product_link_selector"):
+        op.add_column(
+            "search_sites",
+            sa.Column("product_link_selector", sa.String(), nullable=True),
+        )
 
 
 def downgrade() -> None:
     """Remove added columns from search_sites."""
-    op.drop_column("search_sites", "product_link_selector")
-    op.drop_column("search_sites", "search_url")
-    op.drop_column("search_sites", "price_selector")
-    op.drop_column("search_sites", "debug_enabled")
+    if column_exists("search_sites", "product_link_selector"):
+        op.drop_column("search_sites", "product_link_selector")
+    if column_exists("search_sites", "search_url"):
+        op.drop_column("search_sites", "search_url")
+    if column_exists("search_sites", "price_selector"):
+        op.drop_column("search_sites", "price_selector")
+    if column_exists("search_sites", "debug_enabled"):
+        op.drop_column("search_sites", "debug_enabled")

From e62ff09667e4897d74022889fa93237a4a335e26 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:25:23 +0000
Subject: [PATCH 10/13] feat: Enhanced stealth JavaScript with comprehensive
 anti-detection

- Multiple webdriver override techniques (property, prototype, getAttribute)
- Realistic navigator.plugins and mimeTypes
- Delete 20+ automation-related window properties (CDP, Selenium, etc.)
- Full chrome object emulation with realistic properties
- WebGL and WebGL2 fingerprinting protection
- Canvas fingerprinting protection with noise injection
- AudioContext fingerprinting protection
- Screen properties spoofing
- iframe contentWindow webdriver hiding
- Battery API mocking
- Hide Playwright/Puppeteer specific objects
---
 app/services/direct_search_service.py | 230 +++++++++++++++++++++-----
 app/services/scraper_service.py       | 229 ++++++++++++++++++++-----
 2 files changed, 381 insertions(+), 78 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 5b0a350..e035571 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -111,68 +111,135 @@ USER_AGENT = USER_AGENT_POOL[0]
 
 # Stealth JavaScript to inject before page load to hide automation
 STEALTH_JS = """
-// Override the navigator.webdriver property
+// ============================================
+// COMPREHENSIVE STEALTH MODE FOR AMAZON
+// ============================================
+
+// 1. Override navigator.webdriver in multiple ways
 Object.defineProperty(navigator, 'webdriver', {
-    get: () => undefined,
+    get: () => false,
     configurable: true
 });
 
-// Override navigator.plugins to have some plugins
+// Also delete from navigator prototype
+delete Object.getPrototypeOf(navigator).webdriver;
+
+// Override the getAttribute method to hide webdriver attribute
+const originalGetAttribute = Element.prototype.getAttribute;
+Element.prototype.getAttribute = function(name) {
+    if (name === 'webdriver') return null;
+    return originalGetAttribute.call(this, name);
+};
+
+// 2. Override navigator.plugins with realistic plugins
 Object.defineProperty(navigator, 'plugins', {
     get: () => {
         const plugins = [
-            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
-            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
-            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '', length: 2 },
+            { name: 'Chromium PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chromium PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 }
         ];
-        plugins.item = (i) => plugins[i];
-        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.item = (i) => plugins[i] || null;
+        plugins.namedItem = (name) => plugins.find(p => p.name === name) || null;
         plugins.refresh = () => {};
+        plugins.length = plugins.length;
         return plugins;
     },
     configurable: true
 });
 
-// Override navigator.languages
+// 3. Override navigator.mimeTypes
+Object.defineProperty(navigator, 'mimeTypes', {
+    get: () => {
+        const mimeTypes = [
+            { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
+            { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' }
+        ];
+        mimeTypes.item = (i) => mimeTypes[i] || null;
+        mimeTypes.namedItem = (name) => mimeTypes.find(m => m.type === name) || null;
+        mimeTypes.length = mimeTypes.length;
+        return mimeTypes;
+    },
+    configurable: true
+});
+
+// 4. Override navigator.languages
 Object.defineProperty(navigator, 'languages', {
     get: () => ['fr-FR', 'fr', 'en-US', 'en'],
     configurable: true
 });
 
-// Override permissions
-const originalQuery = window.navigator.permissions.query;
-window.navigator.permissions.query = (parameters) => (
-    parameters.name === 'notifications' ?
-        Promise.resolve({ state: Notification.permission }) :
-        originalQuery(parameters)
-);
+// 5. Override permissions API
+if (window.navigator.permissions) {
+    const originalQuery = window.navigator.permissions.query;
+    window.navigator.permissions.query = (parameters) => {
+        if (parameters.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission, onchange: null });
+        }
+        return originalQuery.call(window.navigator.permissions, parameters);
+    };
+}
 
-// Hide automation-related properties
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+// 6. Hide automation-related properties (Chromium DevTools Protocol)
+const propsToDelete = [
+    'cdc_adoQpoasnfa76pfcZLmcfl_Array',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Promise',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Symbol',
+    '__webdriver_evaluate',
+    '__selenium_evaluate',
+    '__webdriver_script_function',
+    '__webdriver_script_func',
+    '__webdriver_script_fn',
+    '__fxdriver_evaluate',
+    '__driver_unwrapped',
+    '__webdriver_unwrapped',
+    '__driver_evaluate',
+    '__selenium_unwrapped',
+    '__fxdriver_unwrapped',
+    '_Selenium_IDE_Recorder',
+    '_selenium',
+    'calledSelenium',
+    '$chrome_asyncScriptInfo',
+    '$cdc_asdjflasutopfhvcZLmcfl_',
+    '$wdc_'
+];
+propsToDelete.forEach(prop => {
+    try { delete window[prop]; } catch(e) {}
+});
 
-// Override chrome object
+// 7. Override chrome object with realistic properties
 window.chrome = {
-    runtime: {},
-    loadTimes: function() {},
-    csi: function() {},
-    app: {}
+    app: {
+        isInstalled: false,
+        InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+        RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+    },
+    runtime: {
+        OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
+        OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
+        PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
+        RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
+        connect: function() { return { onDisconnect: { addListener: function() {} }, onMessage: { addListener: function() {} }, postMessage: function() {} }; },
+        sendMessage: function() {}
+    },
+    csi: function() { return {}; },
+    loadTimes: function() { return { requestTime: Date.now() / 1000, startLoadTime: Date.now() / 1000, firstPaintAfterLoadTime: 0, firstPaintTime: Date.now() / 1000, navigationType: 'navigate' }; }
 };
 
-// Fake WebGL vendor and renderer
+// 8. WebGL fingerprinting protection
 const getParameterProxyHandler = {
     apply: function(target, ctx, args) {
-        if (args[0] === 37445) {
-            return 'Intel Inc.';
-        }
-        if (args[0] === 37446) {
-            return 'Intel Iris OpenGL Engine';
-        }
+        if (args[0] === 37445) return 'Intel Inc.';
+        if (args[0] === 37446) return 'Intel Iris OpenGL Engine';
+        if (args[0] === 7937) return 'WebKit';
+        if (args[0] === 7936) return 'WebKit WebGL';
         return Reflect.apply(target, ctx, args);
     }
 };
-
 try {
     const canvas = document.createElement('canvas');
     const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
@@ -180,32 +247,117 @@ try {
         const getParameter = gl.getParameter.bind(gl);
         gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
     }
+    const gl2 = canvas.getContext('webgl2');
+    if (gl2) {
+        const getParameter2 = gl2.getParameter.bind(gl2);
+        gl2.getParameter = new Proxy(getParameter2, getParameterProxyHandler);
+    }
 } catch(e) {}
 
-// Override connection rtt (reduces fingerprinting)
+// 9. Canvas fingerprinting protection
+const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
+HTMLCanvasElement.prototype.toDataURL = function(type) {
+    if (type === 'image/png' && this.width === 220 && this.height === 30) {
+        // Likely fingerprinting attempt, add noise
+        const context = this.getContext('2d');
+        if (context) {
+            const imageData = context.getImageData(0, 0, this.width, this.height);
+            for (let i = 0; i < imageData.data.length; i += 4) {
+                imageData.data[i] = imageData.data[i] ^ (Math.random() * 2);
+            }
+            context.putImageData(imageData, 0, 0);
+        }
+    }
+    return originalToDataURL.apply(this, arguments);
+};
+
+// 10. AudioContext fingerprinting protection
+if (window.AudioContext || window.webkitAudioContext) {
+    const AudioContext = window.AudioContext || window.webkitAudioContext;
+    const originalCreateOscillator = AudioContext.prototype.createOscillator;
+    AudioContext.prototype.createOscillator = function() {
+        const oscillator = originalCreateOscillator.apply(this, arguments);
+        oscillator.frequency.value = oscillator.frequency.value + (Math.random() * 0.0001);
+        return oscillator;
+    };
+}
+
+// 11. Override connection properties
 Object.defineProperty(navigator, 'connection', {
     get: () => ({
         effectiveType: '4g',
-        rtt: 50,
-        downlink: 10,
+        rtt: 50 + Math.floor(Math.random() * 50),
+        downlink: 10 + Math.random() * 5,
         saveData: false
     }),
     configurable: true
 });
 
-// Override hardware concurrency
+// 12. Override hardware properties
 Object.defineProperty(navigator, 'hardwareConcurrency', {
     get: () => 8,
     configurable: true
 });
 
-// Override device memory
 Object.defineProperty(navigator, 'deviceMemory', {
     get: () => 8,
     configurable: true
 });
 
-console.log('Stealth mode activated');
+// 13. Override screen properties
+Object.defineProperty(screen, 'colorDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+Object.defineProperty(screen, 'pixelDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+// 14. Spoof Notification API
+if (window.Notification) {
+    Object.defineProperty(Notification, 'permission', {
+        get: () => 'default',
+        configurable: true
+    });
+}
+
+// 15. Override iframe contentWindow checks
+const originalContentWindow = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow');
+Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
+    get: function() {
+        const win = originalContentWindow.get.call(this);
+        if (win) {
+            try {
+                Object.defineProperty(win.navigator, 'webdriver', {
+                    get: () => false,
+                    configurable: true
+                });
+            } catch(e) {}
+        }
+        return win;
+    }
+});
+
+// 16. Mock Battery API
+if (navigator.getBattery) {
+    navigator.getBattery = () => Promise.resolve({
+        charging: true,
+        chargingTime: 0,
+        dischargingTime: Infinity,
+        level: 1,
+        addEventListener: () => {},
+        removeEventListener: () => {}
+    });
+}
+
+// 17. Hide Playwright/Puppeteer specific objects
+delete window.__playwright;
+delete window.__pw_manual;
+delete window.__PW_inspect;
+
+console.log('Advanced stealth mode activated');
 """
 
 
diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py
index e0e1d18..ffd0833 100644
--- a/app/services/scraper_service.py
+++ b/app/services/scraper_service.py
@@ -43,68 +43,135 @@ AMAZON_BLOCK_INDICATORS = [
 
 # Stealth JavaScript to inject before page load to hide automation
 STEALTH_JS = """
-// Override the navigator.webdriver property
+// ============================================
+// COMPREHENSIVE STEALTH MODE FOR AMAZON
+// ============================================
+
+// 1. Override navigator.webdriver in multiple ways
 Object.defineProperty(navigator, 'webdriver', {
-    get: () => undefined,
+    get: () => false,
     configurable: true
 });
 
-// Override navigator.plugins to have some plugins
+// Also delete from navigator prototype
+delete Object.getPrototypeOf(navigator).webdriver;
+
+// Override the getAttribute method to hide webdriver attribute
+const originalGetAttribute = Element.prototype.getAttribute;
+Element.prototype.getAttribute = function(name) {
+    if (name === 'webdriver') return null;
+    return originalGetAttribute.call(this, name);
+};
+
+// 2. Override navigator.plugins with realistic plugins
 Object.defineProperty(navigator, 'plugins', {
     get: () => {
         const plugins = [
-            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
-            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
-            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '', length: 2 },
+            { name: 'Chromium PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chromium PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 }
         ];
-        plugins.item = (i) => plugins[i];
-        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.item = (i) => plugins[i] || null;
+        plugins.namedItem = (name) => plugins.find(p => p.name === name) || null;
         plugins.refresh = () => {};
+        plugins.length = plugins.length;
         return plugins;
     },
     configurable: true
 });
 
-// Override navigator.languages
+// 3. Override navigator.mimeTypes
+Object.defineProperty(navigator, 'mimeTypes', {
+    get: () => {
+        const mimeTypes = [
+            { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
+            { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' }
+        ];
+        mimeTypes.item = (i) => mimeTypes[i] || null;
+        mimeTypes.namedItem = (name) => mimeTypes.find(m => m.type === name) || null;
+        mimeTypes.length = mimeTypes.length;
+        return mimeTypes;
+    },
+    configurable: true
+});
+
+// 4. Override navigator.languages
 Object.defineProperty(navigator, 'languages', {
     get: () => ['fr-FR', 'fr', 'en-US', 'en'],
     configurable: true
 });
 
-// Override permissions
-const originalQuery = window.navigator.permissions.query;
-window.navigator.permissions.query = (parameters) => (
-    parameters.name === 'notifications' ?
-        Promise.resolve({ state: Notification.permission }) :
-        originalQuery(parameters)
-);
+// 5. Override permissions API
+if (window.navigator.permissions) {
+    const originalQuery = window.navigator.permissions.query;
+    window.navigator.permissions.query = (parameters) => {
+        if (parameters.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission, onchange: null });
+        }
+        return originalQuery.call(window.navigator.permissions, parameters);
+    };
+}
 
-// Hide automation-related properties
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
-delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+// 6. Hide automation-related properties (Chromium DevTools Protocol)
+const propsToDelete = [
+    'cdc_adoQpoasnfa76pfcZLmcfl_Array',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Promise',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Symbol',
+    '__webdriver_evaluate',
+    '__selenium_evaluate',
+    '__webdriver_script_function',
+    '__webdriver_script_func',
+    '__webdriver_script_fn',
+    '__fxdriver_evaluate',
+    '__driver_unwrapped',
+    '__webdriver_unwrapped',
+    '__driver_evaluate',
+    '__selenium_unwrapped',
+    '__fxdriver_unwrapped',
+    '_Selenium_IDE_Recorder',
+    '_selenium',
+    'calledSelenium',
+    '$chrome_asyncScriptInfo',
+    '$cdc_asdjflasutopfhvcZLmcfl_',
+    '$wdc_'
+];
+propsToDelete.forEach(prop => {
+    try { delete window[prop]; } catch(e) {}
+});
 
-// Override chrome object
+// 7. Override chrome object with realistic properties
 window.chrome = {
-    runtime: {},
-    loadTimes: function() {},
-    csi: function() {},
-    app: {}
+    app: {
+        isInstalled: false,
+        InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+        RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+    },
+    runtime: {
+        OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
+        OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
+        PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
+        RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
+        connect: function() { return { onDisconnect: { addListener: function() {} }, onMessage: { addListener: function() {} }, postMessage: function() {} }; },
+        sendMessage: function() {}
+    },
+    csi: function() { return {}; },
+    loadTimes: function() { return { requestTime: Date.now() / 1000, startLoadTime: Date.now() / 1000, firstPaintAfterLoadTime: 0, firstPaintTime: Date.now() / 1000, navigationType: 'navigate' }; }
 };
 
-// Fake WebGL vendor and renderer
+// 8. WebGL fingerprinting protection
 const getParameterProxyHandler = {
     apply: function(target, ctx, args) {
-        if (args[0] === 37445) {
-            return 'Intel Inc.';
-        }
-        if (args[0] === 37446) {
-            return 'Intel Iris OpenGL Engine';
-        }
+        if (args[0] === 37445) return 'Intel Inc.';
+        if (args[0] === 37446) return 'Intel Iris OpenGL Engine';
+        if (args[0] === 7937) return 'WebKit';
+        if (args[0] === 7936) return 'WebKit WebGL';
         return Reflect.apply(target, ctx, args);
     }
 };
-
 try {
     const canvas = document.createElement('canvas');
     const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
@@ -112,32 +179,116 @@ try {
         const getParameter = gl.getParameter.bind(gl);
         gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
     }
+    const gl2 = canvas.getContext('webgl2');
+    if (gl2) {
+        const getParameter2 = gl2.getParameter.bind(gl2);
+        gl2.getParameter = new Proxy(getParameter2, getParameterProxyHandler);
+    }
 } catch(e) {}
 
-// Override connection rtt (reduces fingerprinting)
+// 9. Canvas fingerprinting protection
+const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
+HTMLCanvasElement.prototype.toDataURL = function(type) {
+    if (type === 'image/png' && this.width === 220 && this.height === 30) {
+        const context = this.getContext('2d');
+        if (context) {
+            const imageData = context.getImageData(0, 0, this.width, this.height);
+            for (let i = 0; i < imageData.data.length; i += 4) {
+                imageData.data[i] = imageData.data[i] ^ (Math.random() * 2);
+            }
+            context.putImageData(imageData, 0, 0);
+        }
+    }
+    return originalToDataURL.apply(this, arguments);
+};
+
+// 10. AudioContext fingerprinting protection
+if (window.AudioContext || window.webkitAudioContext) {
+    const AudioContext = window.AudioContext || window.webkitAudioContext;
+    const originalCreateOscillator = AudioContext.prototype.createOscillator;
+    AudioContext.prototype.createOscillator = function() {
+        const oscillator = originalCreateOscillator.apply(this, arguments);
+        oscillator.frequency.value = oscillator.frequency.value + (Math.random() * 0.0001);
+        return oscillator;
+    };
+}
+
+// 11. Override connection properties
 Object.defineProperty(navigator, 'connection', {
     get: () => ({
         effectiveType: '4g',
-        rtt: 50,
-        downlink: 10,
+        rtt: 50 + Math.floor(Math.random() * 50),
+        downlink: 10 + Math.random() * 5,
         saveData: false
     }),
     configurable: true
 });
 
-// Override hardware concurrency
+// 12. Override hardware properties
 Object.defineProperty(navigator, 'hardwareConcurrency', {
     get: () => 8,
     configurable: true
 });
 
-// Override device memory
 Object.defineProperty(navigator, 'deviceMemory', {
     get: () => 8,
     configurable: true
 });
 
-console.log('Stealth mode activated');
+// 13. Override screen properties
+Object.defineProperty(screen, 'colorDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+Object.defineProperty(screen, 'pixelDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+// 14. Spoof Notification API
+if (window.Notification) {
+    Object.defineProperty(Notification, 'permission', {
+        get: () => 'default',
+        configurable: true
+    });
+}
+
+// 15. Override iframe contentWindow checks
+const originalContentWindow = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow');
+Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
+    get: function() {
+        const win = originalContentWindow.get.call(this);
+        if (win) {
+            try {
+                Object.defineProperty(win.navigator, 'webdriver', {
+                    get: () => false,
+                    configurable: true
+                });
+            } catch(e) {}
+        }
+        return win;
+    }
+});
+
+// 16. Mock Battery API
+if (navigator.getBattery) {
+    navigator.getBattery = () => Promise.resolve({
+        charging: true,
+        chargingTime: 0,
+        dischargingTime: Infinity,
+        level: 1,
+        addEventListener: () => {},
+        removeEventListener: () => {}
+    });
+}
+
+// 17. Hide Playwright/Puppeteer specific objects
+delete window.__playwright;
+delete window.__pw_manual;
+delete window.__PW_inspect;
+
+console.log('Advanced stealth mode activated');
 """
 
 

From 10ec21f4f5299d3b874c0c1e0b33c7198e627c3a Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:30:55 +0000
Subject: [PATCH 11/13] feat: Add proxy support for Amazon scraping

- Add AMAZON_PROXY_URL and AMAZON_PROXY_LIST environment variables
- Support single proxy or rotating proxy list (comma-separated)
- Apply proxy only for Amazon requests
- Log proxy usage (hiding credentials)
---
 app/services/direct_search_service.py | 43 ++++++++++++++++++++-------
 app/services/scraper_service.py       | 42 +++++++++++++++++++-------
 2 files changed, 65 insertions(+), 20 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index e035571..6317c25 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -18,6 +18,20 @@ logger = logging.getLogger(__name__)
 
 BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 
+# Proxy configuration for Amazon (optional)
+# Format: "http://user:pass@host:port" or "http://host:port"
+AMAZON_PROXY_URL = os.getenv("AMAZON_PROXY_URL", "")
+# If you have multiple proxies, separate them with commas
+AMAZON_PROXY_LIST = [p.strip() for p in os.getenv("AMAZON_PROXY_LIST", "").split(",") if p.strip()]
+
+def _get_amazon_proxy() -> str | None:
+    """Get a proxy for Amazon requests (random from list or single)"""
+    if AMAZON_PROXY_LIST:
+        return random.choice(AMAZON_PROXY_LIST)
+    if AMAZON_PROXY_URL:
+        return AMAZON_PROXY_URL
+    return None
+
 # Constants
 MIN_TITLE_LENGTH = 3
 MIN_LINK_TEXT_LENGTH = 5
@@ -940,17 +954,26 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             if is_amazon:
                 extra_headers["Referer"] = "https://www.google.fr/"
 
+            # Prepare context options
+            context_options = {
+                "viewport": {"width": 1920, "height": 1080},
+                "user_agent": current_user_agent,
+                "locale": "fr-FR",
+                "timezone_id": "Europe/Paris",
+                "extra_http_headers": extra_headers,
+                "java_script_enabled": True,
+                "bypass_csp": True,
+            }
+
+            # Add proxy for Amazon if configured
+            if is_amazon:
+                proxy_url = _get_amazon_proxy()
+                if proxy_url:
+                    logger.info(f"Using proxy for Amazon: {proxy_url.split('@')[-1] if '@' in proxy_url else proxy_url}")
+                    context_options["proxy"] = {"server": proxy_url}
+
             # Créer un nouveau contexte isolé pour ce site
-            context = await browser.new_context(
-                viewport={"width": 1920, "height": 1080},
-                user_agent=current_user_agent,
-                locale="fr-FR",
-                timezone_id="Europe/Paris",
-                extra_http_headers=extra_headers,
-                # Additional stealth options
-                java_script_enabled=True,
-                bypass_csp=True,  # Bypass Content Security Policy for script injection
-            )
+            context = await browser.new_context(**context_options)
 
             # Inject stealth script before any page loads (especially for Amazon)
             if is_amazon:
diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py
index ffd0833..7ffd465 100644
--- a/app/services/scraper_service.py
+++ b/app/services/scraper_service.py
@@ -12,6 +12,19 @@ logger = logging.getLogger(__name__)
 
 BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 
+# Proxy configuration for Amazon (optional)
+# Format: "http://user:pass@host:port" or "http://host:port"
+AMAZON_PROXY_URL = os.getenv("AMAZON_PROXY_URL", "")
+AMAZON_PROXY_LIST = [p.strip() for p in os.getenv("AMAZON_PROXY_LIST", "").split(",") if p.strip()]
+
+def _get_amazon_proxy() -> str | None:
+    """Get a proxy for Amazon requests"""
+    if AMAZON_PROXY_LIST:
+        return random.choice(AMAZON_PROXY_LIST)
+    if AMAZON_PROXY_URL:
+        return AMAZON_PROXY_URL
+    return None
+
 # Nombre de tentatives pour le scraping
 MAX_RETRIES = 2
 RETRY_DELAY = 3  # secondes
@@ -513,16 +526,25 @@ class ScraperService:
                 extra_headers["Referer"] = "https://www.google.fr/"
 
             try:
-                context = await current_browser.new_context(
-                    viewport={"width": 1920, "height": 1080},
-                    user_agent=current_user_agent,
-                    locale="fr-FR",
-                    timezone_id="Europe/Paris",
-                    extra_http_headers=extra_headers,
-                    # Additional stealth options
-                    java_script_enabled=True,
-                    bypass_csp=True,  # Bypass Content Security Policy for script injection
-                )
+                # Prepare context options
+                context_options = {
+                    "viewport": {"width": 1920, "height": 1080},
+                    "user_agent": current_user_agent,
+                    "locale": "fr-FR",
+                    "timezone_id": "Europe/Paris",
+                    "extra_http_headers": extra_headers,
+                    "java_script_enabled": True,
+                    "bypass_csp": True,
+                }
+
+                # Add proxy for Amazon if configured
+                if is_amazon:
+                    proxy_url = _get_amazon_proxy()
+                    if proxy_url:
+                        logger.info(f"Using proxy for Amazon: {proxy_url.split('@')[-1] if '@' in proxy_url else proxy_url}")
+                        context_options["proxy"] = {"server": proxy_url}
+
+                context = await current_browser.new_context(**context_options)
 
                 # Inject stealth script before any page loads (especially for Amazon)
                 if is_amazon:

From 698dbaa0293c335fa2c0552fe2c4790f62aaeb25 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Nov 2025 08:39:11 +0000
Subject: [PATCH 12/13] feat: Add dedicated Amazon extraction without generic
 filters

- Create _find_amazon_product_links() function for Amazon-specific extraction
- Use ASIN-based deduplication for Amazon products
- Bypass all generic filters (_is_non_product_url, _is_navigation_link) for Amazon
- Multiple fallback methods for Amazon product detection:
  - data-component-type="s-product-image"
  - .s-result-item links
  - All /dp/ links in main content
- Clean Amazon URLs by removing tracking parameters
- Extract title from img alt as fallback for Amazon

This separates Amazon's extraction logic from the generic e-commerce filters
that were designed for French sites like Cdiscount, Fnac, etc.
---
 app/services/direct_search_service.py | 220 +++++++++++++++++++-------
 1 file changed, 164 insertions(+), 56 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 6317c25..5728373 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -1054,74 +1054,119 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             results = []
             seen_urls = set()
 
-            # Trouver les liens produits
-            links = []
-            if product_selector:
-                # Essayer plusieurs sélecteurs séparés par virgule
-                selectors = [s.strip() for s in product_selector.split(",")]
-                for sel in selectors:
-                    try:
-                        found = soup.select(sel)
-                        links.extend(found)
-                    except Exception:
+            # ===== AMAZON: Utiliser extraction dédiée sans les filtres génériques =====
+            if is_amazon:
+                logger.info(f"{domain}: Utilisation extraction Amazon dédiée")
+                links = _find_amazon_product_links(soup)
+
+                for link in links:
+                    href = link.get("href", "")
+                    if not href:
                         continue
-                logger.info(f"{domain}: {len(links)} liens trouvés avec sélecteur configuré")
 
-            # Fallback: détection générique si le sélecteur n'a rien trouvé
-            if not links:
-                logger.info(f"{domain}: Sélecteur configuré n'a rien trouvé, utilisation détection générique")
-                links = _find_product_links(soup, domain)
-                logger.info(f"{domain}: {len(links)} liens trouvés avec détection générique")
+                    # Construire l'URL complète pour Amazon
+                    if href.startswith("/"):
+                        href = f"https://www.amazon.fr{href}"
+                    elif not href.startswith("http"):
+                        href = urljoin(final_url, href)
 
-            for link in links:
-                href = link.get("href", "")
-                if not href:
-                    continue
+                    # Nettoyer l'URL Amazon (enlever les paramètres de tracking)
+                    if "?" in href:
+                        href = href.split("?")[0]
 
-                # Construire l'URL complète
-                if href.startswith("/"):
-                    href = f"https://{domain}{href}"
-                elif not href.startswith("http"):
-                    href = urljoin(final_url, href)
+                    # Éviter les doublons
+                    if href in seen_urls:
+                        continue
+                    seen_urls.add(href)
 
-                # Nettoyer l'URL
-                href = _clean_product_url(href, domain)
+                    # Extraire le titre
+                    title = _extract_title(link)
+                    if not title or len(title) < MIN_TITLE_LENGTH:
+                        # Pour Amazon, essayer aussi alt de l'image
+                        img = link.find("img")
+                        if img and img.get("alt"):
+                            title = img.get("alt").strip()
 
-                # Éviter les doublons
-                if href in seen_urls:
-                    continue
-                seen_urls.add(href)
+                    if not title or len(title) < MIN_TITLE_LENGTH:
+                        continue
 
-                # Vérifier que c'est bien un lien vers ce domaine
-                url_domain = _extract_domain(href)
-                if domain not in url_domain and url_domain not in domain:
-                    # logger.debug(f"Ignored external link: {href}")
-                    continue
+                    results.append(SearchResult(
+                        url=href,
+                        title=title,
+                        snippet="",
+                        source=domain,
+                    ))
 
-                # Filtrer les liens non-produits
-                if _is_non_product_url(href):
-                    # logger.debug(f"Ignored non-product link: {href}")
-                    continue
+                    if len(results) >= max_results:
+                        break
 
-                # Extraire le titre
-                title = _extract_title(link)
-                if not title or len(title) < MIN_TITLE_LENGTH:
-                    # logger.debug(f"Ignored link with no title: {href}")
-                    continue
+                logger.info(f"{domain}: {len(results)} résultats Amazon extraits")
 
-                results.append(SearchResult(
-                    url=href,
-                    title=title,
-                    snippet="",
-                    source=domain,
-                    # On ne peut pas extraire le prix facilement ici sans scraper chaque page
-                    # Le scraping détaillé se fera dans la phase 2
-                ))
+            # ===== AUTRES SITES: Utiliser la logique générique avec filtres =====
+            else:
+                # Trouver les liens produits
+                links = []
+                if product_selector:
+                    # Essayer plusieurs sélecteurs séparés par virgule
+                    selectors = [s.strip() for s in product_selector.split(",")]
+                    for sel in selectors:
+                        try:
+                            found = soup.select(sel)
+                            links.extend(found)
+                        except Exception:
+                            continue
+                    logger.info(f"{domain}: {len(links)} liens trouvés avec sélecteur configuré")
 
-                if len(results) >= max_results:
-                    break
+                # Fallback: détection générique si le sélecteur n'a rien trouvé
+                if not links:
+                    logger.info(f"{domain}: Sélecteur configuré n'a rien trouvé, utilisation détection générique")
+                    links = _find_product_links(soup, domain)
+                    logger.info(f"{domain}: {len(links)} liens trouvés avec détection générique")
 
-            logger.info(f"{domain}: {len(results)} résultats extraits")
+                for link in links:
+                    href = link.get("href", "")
+                    if not href:
+                        continue
+
+                    # Construire l'URL complète
+                    if href.startswith("/"):
+                        href = f"https://{domain}{href}"
+                    elif not href.startswith("http"):
+                        href = urljoin(final_url, href)
+
+                    # Nettoyer l'URL
+                    href = _clean_product_url(href, domain)
+
+                    # Éviter les doublons
+                    if href in seen_urls:
+                        continue
+                    seen_urls.add(href)
+
+                    # Vérifier que c'est bien un lien vers ce domaine
+                    url_domain = _extract_domain(href)
+                    if domain not in url_domain and url_domain not in domain:
+                        continue
+
+                    # Filtrer les liens non-produits (pas pour Amazon qui a sa propre logique)
+                    if _is_non_product_url(href):
+                        continue
+
+                    # Extraire le titre
+                    title = _extract_title(link)
+                    if not title or len(title) < MIN_TITLE_LENGTH:
+                        continue
+
+                    results.append(SearchResult(
+                        url=href,
+                        title=title,
+                        snippet="",
+                        source=domain,
+                    ))
+
+                    if len(results) >= max_results:
+                        break
+
+                logger.info(f"{domain}: {len(results)} résultats extraits")
 
             # If we got results, success! Return them
             if len(results) > 0:
@@ -1215,6 +1260,69 @@ def _get_default_wait_selector(domain: str) -> str | None:
     return None
 
 
+def _find_amazon_product_links(soup: BeautifulSoup) -> list:
+    """
+    Extraction dédiée pour Amazon - simple et directe, sans tous les filtres génériques.
+    Amazon utilise des structures spécifiques qui ne nécessitent pas le filtrage complexe
+    utilisé pour les autres sites e-commerce.
+    """
+    links = []
+
+    # Amazon utilise des data-attributes spécifiques pour les résultats de recherche
+    # Méthode 1: Liens avec data-component-type="s-product-image"
+    for item in soup.select('[data-component-type="s-product-image"] a'):
+        href = item.get("href", "")
+        if href and "/dp/" in href:
+            links.append(item)
+
+    # Méthode 2: Liens dans les cartes de produits Amazon
+    if not links:
+        for item in soup.select('.s-result-item a.a-link-normal[href*="/dp/"]'):
+            links.append(item)
+
+    # Méthode 3: Tous les liens avec /dp/ dans le main content
+    if not links:
+        main_content = soup.select_one('#search, .s-main-slot, #s-results-list-atf')
+        if main_content:
+            for a in main_content.find_all('a', href=True):
+                href = a.get('href', '')
+                if '/dp/' in href and href not in [l.get('href', '') for l in links]:
+                    links.append(a)
+
+    # Méthode 4: Fallback - tous les liens /dp/ sur la page
+    if not links:
+        for a in soup.find_all('a', href=True):
+            href = a.get('href', '')
+            if '/dp/' in href or '/gp/product/' in href:
+                # Ignorer les liens de navigation Amazon
+                if '/ref=nb_sb' not in href and '/ref=cs_' not in href:
+                    links.append(a)
+
+    # Dédupliquer par ASIN (l'identifiant produit Amazon)
+    seen_asins = set()
+    unique_links = []
+    for link in links:
+        href = link.get('href', '')
+        # Extraire l'ASIN du lien
+        if '/dp/' in href:
+            parts = href.split('/dp/')
+            if len(parts) > 1:
+                asin = parts[1].split('/')[0].split('?')[0]
+                if asin and asin not in seen_asins and len(asin) == 10:
+                    seen_asins.add(asin)
+                    unique_links.append(link)
+        elif '/gp/product/' in href:
+            parts = href.split('/gp/product/')
+            if len(parts) > 1:
+                asin = parts[1].split('/')[0].split('?')[0]
+                if asin and asin not in seen_asins and len(asin) == 10:
+                    seen_asins.add(asin)
+                    unique_links.append(link)
+
+    logger.info(f"Amazon: {len(unique_links)} produits uniques trouvés (ASINs: {list(seen_asins)[:5]}...)")
+    return unique_links
+
+
 def _find_product_links(soup: BeautifulSoup, domain: str) -> list:
     """Trouve les liens produits de manière générique"""
     links = []

From 3caea8316b1e5cbd88884ab107d4bdc1fbeff230 Mon Sep 17 00:00:00 2001
From: Michael SCHAL <michaelschal@gmail.com>
Date: Fri, 28 Nov 2025 23:53:45 +0100
Subject: [PATCH 13/13] feat: Add Playwright-based web scraping service with
 advanced stealth techniques and proxy support, including Amazon-specific
 logic.

---
 app/services/direct_search_service.py | 20 ++++++---
 app/services/scraper_service.py       |  9 ++++
 debug_amazon.py                       | 60 +++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 6 deletions(-)
 create mode 100644 debug_amazon.py

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 5728373..d2ea58f 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -41,18 +41,14 @@ USER_AGENT_POOL = [
     # Chrome Windows - Latest versions
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
     # Chrome Mac
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
     # Firefox Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
-    # Firefox Mac
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
     # Edge Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    # Linux
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 ]
 
 # Amazon-specific blocking indicators
@@ -992,6 +988,18 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             if is_amazon:
                 await _random_delay(1.0, 3.0)
 
+            # Warmup for Amazon: Visit home page first to get cookies (only on first attempt)
+            if is_amazon and attempt == 0:
+                try:
+                    logger.info(f"Amazon Warmup: Visiting home page...")
+                    # Use a shorter timeout for warmup
+                    await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
+                    await asyncio.sleep(random.uniform(2.0, 4.0))
+                    # Simulate a bit of mouse movement
+                    await _simulate_human_behavior(page)
+                except Exception as e:
+                    logger.warning(f"Amazon warmup failed (non-critical): {e}")
+
             # Naviguer vers la page de recherche
             await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
 
diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py
index 7ffd465..b36551b 100644
--- a/app/services/scraper_service.py
+++ b/app/services/scraper_service.py
@@ -40,6 +40,7 @@ USER_AGENT_POOL = [
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 ]
 
 # Amazon blocking indicators
@@ -564,6 +565,14 @@ class ScraperService:
                 # Add random delay before navigation for Amazon
                 if is_amazon:
                     await asyncio.sleep(random.uniform(1.0, 3.0))
+                    
+                    # Warmup: Visit home page first
+                    try:
+                        logger.info("Amazon Warmup: Visiting home page...")
+                        await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
+                        await asyncio.sleep(random.uniform(2.0, 4.0))
+                    except Exception as e:
+                        logger.warning(f"Amazon warmup failed: {e}")
 
                 logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
                 response = None
diff --git a/debug_amazon.py b/debug_amazon.py
new file mode 100644
index 0000000..2497fbf
--- /dev/null
+++ b/debug_amazon.py
@@ -0,0 +1,60 @@
+import asyncio
+import logging
+import os
+from playwright.async_api import async_playwright
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
+
+async def debug_amazon():
+    logger.info("Starting Amazon Debug Script")
+    
+    async with async_playwright() as p:
+        try:
+            logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+            browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
+            
+            # Use a very standard, recent User-Agent
+            user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            
+            context = await browser.new_context(
+                user_agent=user_agent,
+                viewport={"width": 1920, "height": 1080},
+                locale="fr-FR",
+                timezone_id="Europe/Paris"
+            )
+            
+            page = await context.new_page()
+            
+            url = "https://www.amazon.fr/s?k=iphone"
+            logger.info(f"Navigating to {url}")
+            
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            
+            if response:
+                status = response.status
+                logger.info(f"Response Status: {status}")
+                
+                content = await page.content()
+                if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content:
+                    logger.error("BLOCK DETECTED: Found blocking message in content")
+                else:
+                    logger.info("No obvious blocking message found")
+                    
+                # Save screenshot
+                await page.screenshot(path="debug_amazon_screenshot.png")
+                logger.info("Screenshot saved to debug_amazon_screenshot.png")
+                
+            else:
+                logger.error("No response received")
+                
+            await browser.close()
+            
+        except Exception as e:
+            logger.error(f"An error occurred: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(debug_amazon())