Merge: resolved conflicts in direct_search_service.py

Dockerfile

@@ -37,8 +37,8 @@ RUN chmod +x docker-entrypoint.sh
 # Copy built frontend static files
 COPY --from=frontend-build /app/frontend/dist /app/static
 
-# Create screenshots directory
+# Create screenshots and debug_dumps directories
-RUN mkdir -p screenshots
+RUN mkdir -p screenshots debug_dumps
 
 # Expose port
 EXPOSE 8555

app/alembic/versions/add_search_sites_columns.py

@@ -0,0 +1,69 @@
+"""Add missing columns to search_sites table
+
+Revision ID: add_search_sites_columns
+Revises: add_notification_channels
+Create Date: 2025-11-24
+
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import inspect
+
+# revision identifiers, used by Alembic.
+revision: str = "add_search_sites_columns"
+down_revision: str | None = "add_notification_channels"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def column_exists(table_name: str, column_name: str) -> bool:
+    """Check if a column exists in a table."""
+    bind = op.get_bind()
+    inspector = inspect(bind)
+    columns = [col["name"] for col in inspector.get_columns(table_name)]
+    return column_name in columns
+
+
+def upgrade() -> None:
+    """Add debug_enabled, price_selector, search_url, product_link_selector columns to search_sites."""
+    # Add debug_enabled column (only if not exists)
+    if not column_exists("search_sites", "debug_enabled"):
+        op.add_column(
+            "search_sites",
+            sa.Column("debug_enabled", sa.Boolean(), server_default="false", nullable=False),
+        )
+
+    # Add price_selector column (only if not exists)
+    if not column_exists("search_sites", "price_selector"):
+        op.add_column(
+            "search_sites",
+            sa.Column("price_selector", sa.String(), nullable=True),
+        )
+
+    # Add search_url column (only if not exists)
+    if not column_exists("search_sites", "search_url"):
+        op.add_column(
+            "search_sites",
+            sa.Column("search_url", sa.String(), nullable=True),
+        )
+
+    # Add product_link_selector column (only if not exists)
+    if not column_exists("search_sites", "product_link_selector"):
+        op.add_column(
+            "search_sites",
+            sa.Column("product_link_selector", sa.String(), nullable=True),
+        )
+
+
+def downgrade() -> None:
+    """Remove added columns from search_sites."""
+    if column_exists("search_sites", "product_link_selector"):
+        op.drop_column("search_sites", "product_link_selector")
+    if column_exists("search_sites", "search_url"):
+        op.drop_column("search_sites", "search_url")
+    if column_exists("search_sites", "price_selector"):
+        op.drop_column("search_sites", "price_selector")
+    if column_exists("search_sites", "debug_enabled"):
+        op.drop_column("search_sites", "debug_enabled")

app/models.py

@@ -94,6 +94,7 @@ class SearchSite(Base):
     is_active: bool = Column(Boolean, default=True)  # type: ignore
     priority: int = Column(Integer, default=0)  # type: ignore  # Ordre d'affichage
     requires_js: bool = Column(Boolean, default=False)  # type: ignore  # Force Browserless si True
+    debug_enabled: bool = Column(Boolean, default=False)  # type: ignore  # Activer le dump HTML pour ce site
     price_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour le prix
     search_url: str | None = Column(String, nullable=True)  # type: ignore  # URL de recherche avec {query} placeholder
     product_link_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour les liens produits

app/routers/debug.py

@@ -18,7 +18,8 @@ router = APIRouter(
 
 logger = logging.getLogger(__name__)
 
-DEBUG_DIR = "debug_dumps"
+# Chemin absolu pour les dumps de débogage (doit correspondre à celui dans direct_search_service.py)
+DEBUG_DIR = "/app/debug_dumps"
 
 class DebugFile(BaseModel):
     filename: str
@@ -90,3 +91,66 @@ def delete_debug_dump(filename: str):
     except Exception as e:
         logger.error(f"Error deleting debug dump {filename}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/dumps/site/{domain}", response_model=List[DebugFile])
+def list_debug_dumps_by_site(domain: str):
+    """List debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return []
+
+    # Clean domain for comparison (remove www. if present)
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    files = []
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            # Check if filename starts with the domain
+            filename_lower = filename.lower()
+            # Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    stat = os.stat(filepath)
+                    files.append(DebugFile(
+                        filename=filename,
+                        size=stat.st_size,
+                        created_at=stat.st_ctime,
+                        created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S")
+                    ))
+
+        # Sort by creation time (newest first)
+        files.sort(key=lambda x: x.created_at, reverse=True)
+        return files
+    except Exception as e:
+        logger.error(f"Error listing debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/dumps/site/{domain}")
+def delete_debug_dumps_by_site(domain: str):
+    """Delete all debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return {"message": "No files found", "deleted": 0}
+
+    # Clean domain for comparison
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    deleted_count = 0
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            filename_lower = filename.lower()
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    os.remove(filepath)
+                    deleted_count += 1
+
+        return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count}
+    except Exception as e:
+        logger.error(f"Error deleting debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

app/routers/search_sites.py

@@ -41,17 +41,17 @@ async def update_site(
 ):
     """
     Met à jour un site de recherche.
-    Permet uniquement de modifier is_active et priority.
+    Permet uniquement de modifier is_active, priority et debug_enabled.
     """
     # Limiter les champs modifiables
-    allowed_fields = {"is_active", "priority"}
+    allowed_fields = {"is_active", "priority", "debug_enabled"}
     update_data = site.model_dump(exclude_unset=True)
     filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields}
 
     if not filtered_data:
         raise HTTPException(
             status_code=400,
-            detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés"
+            detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés"
         )
 
     updated = search_service.update_site(db, site_id, filtered_data)

app/schemas.py

@@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel):
     is_active: bool = True
     priority: int = 0
     requires_js: bool = False
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None  # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query}
     product_link_selector: str | None = None  # Sélecteur CSS pour les liens produits
@@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel):
     is_active: bool | None = None
     priority: int | None = None
     requires_js: bool | None = None
+    debug_enabled: bool | None = None  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None
@@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel):
     is_active: bool
     priority: int
     requires_js: bool
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None

app/services/scraper_service.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import os
+import random
 import re
 from urllib.parse import urlparse, urlunparse
 
@@ -11,10 +12,318 @@ logger = logging.getLogger(__name__)
 
 BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 
+# Proxy configuration for Amazon (optional)
+# Format: "http://user:pass@host:port" or "http://host:port"
+AMAZON_PROXY_URL = os.getenv("AMAZON_PROXY_URL", "")
+AMAZON_PROXY_LIST = [p.strip() for p in os.getenv("AMAZON_PROXY_LIST", "").split(",") if p.strip()]
+
+def _get_amazon_proxy() -> str | None:
+    """Get a proxy for Amazon requests"""
+    if AMAZON_PROXY_LIST:
+        return random.choice(AMAZON_PROXY_LIST)
+    if AMAZON_PROXY_URL:
+        return AMAZON_PROXY_URL
+    return None
+
 # Nombre de tentatives pour le scraping
 MAX_RETRIES = 2
 RETRY_DELAY = 3  # secondes
 
+# Amazon-specific configuration
+AMAZON_MAX_RETRIES = 4
+AMAZON_BASE_DELAY = 3.0
+
+# Pool of realistic User-Agents for rotation
+USER_AGENT_POOL = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+]
+
+# Amazon blocking indicators
+AMAZON_BLOCK_INDICATORS = [
+    "Toutes nos excuses",
+    "Sorry, something went wrong",
+    "api-services-support@amazon.com",
+    "Enter the characters you see below",
+    "Saisissez les caractères",
+    "Service Unavailable",
+    "robot check",
+    "automated access",
+]
+
+# Stealth JavaScript to inject before page load to hide automation
+STEALTH_JS = """
+// ============================================
+// COMPREHENSIVE STEALTH MODE FOR AMAZON
+// ============================================
+
+// 1. Override navigator.webdriver in multiple ways
+Object.defineProperty(navigator, 'webdriver', {
+    get: () => false,
+    configurable: true
+});
+
+// Also delete from navigator prototype
+delete Object.getPrototypeOf(navigator).webdriver;
+
+// Override the getAttribute method to hide webdriver attribute
+const originalGetAttribute = Element.prototype.getAttribute;
+Element.prototype.getAttribute = function(name) {
+    if (name === 'webdriver') return null;
+    return originalGetAttribute.call(this, name);
+};
+
+// 2. Override navigator.plugins with realistic plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '', length: 2 },
+            { name: 'Chromium PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
+            { name: 'Chromium PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 }
+        ];
+        plugins.item = (i) => plugins[i] || null;
+        plugins.namedItem = (name) => plugins.find(p => p.name === name) || null;
+        plugins.refresh = () => {};
+        plugins.length = plugins.length;
+        return plugins;
+    },
+    configurable: true
+});
+
+// 3. Override navigator.mimeTypes
+Object.defineProperty(navigator, 'mimeTypes', {
+    get: () => {
+        const mimeTypes = [
+            { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
+            { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' }
+        ];
+        mimeTypes.item = (i) => mimeTypes[i] || null;
+        mimeTypes.namedItem = (name) => mimeTypes.find(m => m.type === name) || null;
+        mimeTypes.length = mimeTypes.length;
+        return mimeTypes;
+    },
+    configurable: true
+});
+
+// 4. Override navigator.languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['fr-FR', 'fr', 'en-US', 'en'],
+    configurable: true
+});
+
+// 5. Override permissions API
+if (window.navigator.permissions) {
+    const originalQuery = window.navigator.permissions.query;
+    window.navigator.permissions.query = (parameters) => {
+        if (parameters.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission, onchange: null });
+        }
+        return originalQuery.call(window.navigator.permissions, parameters);
+    };
+}
+
+// 6. Hide automation-related properties (Chromium DevTools Protocol)
+const propsToDelete = [
+    'cdc_adoQpoasnfa76pfcZLmcfl_Array',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Promise',
+    'cdc_adoQpoasnfa76pfcZLmcfl_Symbol',
+    '__webdriver_evaluate',
+    '__selenium_evaluate',
+    '__webdriver_script_function',
+    '__webdriver_script_func',
+    '__webdriver_script_fn',
+    '__fxdriver_evaluate',
+    '__driver_unwrapped',
+    '__webdriver_unwrapped',
+    '__driver_evaluate',
+    '__selenium_unwrapped',
+    '__fxdriver_unwrapped',
+    '_Selenium_IDE_Recorder',
+    '_selenium',
+    'calledSelenium',
+    '$chrome_asyncScriptInfo',
+    '$cdc_asdjflasutopfhvcZLmcfl_',
+    '$wdc_'
+];
+propsToDelete.forEach(prop => {
+    try { delete window[prop]; } catch(e) {}
+});
+
+// 7. Override chrome object with realistic properties
+window.chrome = {
+    app: {
+        isInstalled: false,
+        InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+        RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+    },
+    runtime: {
+        OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
+        OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
+        PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+        PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
+        RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
+        connect: function() { return { onDisconnect: { addListener: function() {} }, onMessage: { addListener: function() {} }, postMessage: function() {} }; },
+        sendMessage: function() {}
+    },
+    csi: function() { return {}; },
+    loadTimes: function() { return { requestTime: Date.now() / 1000, startLoadTime: Date.now() / 1000, firstPaintAfterLoadTime: 0, firstPaintTime: Date.now() / 1000, navigationType: 'navigate' }; }
+};
+
+// 8. WebGL fingerprinting protection
+const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+        if (args[0] === 37445) return 'Intel Inc.';
+        if (args[0] === 37446) return 'Intel Iris OpenGL Engine';
+        if (args[0] === 7937) return 'WebKit';
+        if (args[0] === 7936) return 'WebKit WebGL';
+        return Reflect.apply(target, ctx, args);
+    }
+};
+try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+        const getParameter = gl.getParameter.bind(gl);
+        gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
+    }
+    const gl2 = canvas.getContext('webgl2');
+    if (gl2) {
+        const getParameter2 = gl2.getParameter.bind(gl2);
+        gl2.getParameter = new Proxy(getParameter2, getParameterProxyHandler);
+    }
+} catch(e) {}
+
+// 9. Canvas fingerprinting protection
+const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
+HTMLCanvasElement.prototype.toDataURL = function(type) {
+    if (type === 'image/png' && this.width === 220 && this.height === 30) {
+        const context = this.getContext('2d');
+        if (context) {
+            const imageData = context.getImageData(0, 0, this.width, this.height);
+            for (let i = 0; i < imageData.data.length; i += 4) {
+                imageData.data[i] = imageData.data[i] ^ (Math.random() * 2);
+            }
+            context.putImageData(imageData, 0, 0);
+        }
+    }
+    return originalToDataURL.apply(this, arguments);
+};
+
+// 10. AudioContext fingerprinting protection
+if (window.AudioContext || window.webkitAudioContext) {
+    const AudioContext = window.AudioContext || window.webkitAudioContext;
+    const originalCreateOscillator = AudioContext.prototype.createOscillator;
+    AudioContext.prototype.createOscillator = function() {
+        const oscillator = originalCreateOscillator.apply(this, arguments);
+        oscillator.frequency.value = oscillator.frequency.value + (Math.random() * 0.0001);
+        return oscillator;
+    };
+}
+
+// 11. Override connection properties
+Object.defineProperty(navigator, 'connection', {
+    get: () => ({
+        effectiveType: '4g',
+        rtt: 50 + Math.floor(Math.random() * 50),
+        downlink: 10 + Math.random() * 5,
+        saveData: false
+    }),
+    configurable: true
+});
+
+// 12. Override hardware properties
+Object.defineProperty(navigator, 'hardwareConcurrency', {
+    get: () => 8,
+    configurable: true
+});
+
+Object.defineProperty(navigator, 'deviceMemory', {
+    get: () => 8,
+    configurable: true
+});
+
+// 13. Override screen properties
+Object.defineProperty(screen, 'colorDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+Object.defineProperty(screen, 'pixelDepth', {
+    get: () => 24,
+    configurable: true
+});
+
+// 14. Spoof Notification API
+if (window.Notification) {
+    Object.defineProperty(Notification, 'permission', {
+        get: () => 'default',
+        configurable: true
+    });
+}
+
+// 15. Override iframe contentWindow checks
+const originalContentWindow = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow');
+Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
+    get: function() {
+        const win = originalContentWindow.get.call(this);
+        if (win) {
+            try {
+                Object.defineProperty(win.navigator, 'webdriver', {
+                    get: () => false,
+                    configurable: true
+                });
+            } catch(e) {}
+        }
+        return win;
+    }
+});
+
+// 16. Mock Battery API
+if (navigator.getBattery) {
+    navigator.getBattery = () => Promise.resolve({
+        charging: true,
+        chargingTime: 0,
+        dischargingTime: Infinity,
+        level: 1,
+        addEventListener: () => {},
+        removeEventListener: () => {}
+    });
+}
+
+// 17. Hide Playwright/Puppeteer specific objects
+delete window.__playwright;
+delete window.__pw_manual;
+delete window.__PW_inspect;
+
+console.log('Advanced stealth mode activated');
+"""
+
+
+def _get_random_user_agent() -> str:
+    """Get a random User-Agent from the pool"""
+    return random.choice(USER_AGENT_POOL)
+
+
+def _is_amazon_url(url: str) -> bool:
+    """Check if URL is an Amazon URL"""
+    return "amazon" in url.lower()
+
+
+def _is_amazon_blocked(html_content: str) -> bool:
+    """Check if Amazon has blocked the request"""
+    html_lower = html_content.lower()
+    for indicator in AMAZON_BLOCK_INDICATORS:
+        if indicator.lower() in html_lower:
+            return True
+    return False
+
 
 def simplify_url(url: str) -> str:
     """
@@ -117,12 +426,23 @@ class ScraperService:
             logger.warning(f"Invalid timeout value: {timeout}, using default 90000")
             timeout = 90000
 
+        # Amazon needs more retries with longer delays
+        is_amazon = _is_amazon_url(url)
+        max_retries = AMAZON_MAX_RETRIES if is_amazon else MAX_RETRIES
+        base_delay = AMAZON_BASE_DELAY if is_amazon else RETRY_DELAY
+
         # Retries avec backoff
         last_error = None
-        for attempt in range(MAX_RETRIES + 1):
+        for attempt in range(max_retries + 1):
             if attempt > 0:
-                logger.info(f"Tentative {attempt + 1}/{MAX_RETRIES + 1} pour {url}")
+                # Exponential backoff with jitter for Amazon
-                await asyncio.sleep(RETRY_DELAY * attempt)
+                if is_amazon:
+                    jitter = random.uniform(0.5, 1.5)
+                    delay = min(base_delay * (2 ** attempt) * jitter, 30.0)
+                else:
+                    delay = base_delay * attempt
+                logger.info(f"Tentative {attempt + 1}/{max_retries + 1} pour {url} (délai: {delay:.1f}s)")
+                await asyncio.sleep(delay)
 
             result = await ScraperService._do_scrape(
                 url=url,
@@ -133,6 +453,7 @@ class ScraperService:
                 text_length=text_length,
                 timeout=timeout,
                 browser=browser,
+                is_amazon=is_amazon,
             )
 
             if result[0] is not None:  # Screenshot réussi
@@ -154,6 +475,7 @@ class ScraperService:
         text_length: int = 0,
         timeout: int = 90000,
         browser=None,
+        is_amazon: bool = False,
     ) -> tuple[str | None, str, bool]:
         """Exécute le scraping réel (appelé par scrape_item avec retries)."""
 
@@ -181,30 +503,80 @@ class ScraperService:
             page = None
             is_available = True  # Par défaut, le produit est disponible
 
-            try:
+            # Use random User-Agent for anti-detection (especially for Amazon)
-                context = await current_browser.new_context(
+            current_user_agent = _get_random_user_agent()
-                    viewport={"width": 1920, "height": 1080},
-                    user_agent=(
-                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                        "AppleWebKit/537.36 (KHTML, like Gecko) "
-                        "Chrome/120.0.0.0 Safari/537.36"
-                    ),
-                    locale="fr-FR",
-                    timezone_id="Europe/Paris",
-                    # Options supplémentaires pour éviter la détection
-                    extra_http_headers={
-                        "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
-                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-                    }
-                )
 
-                # Stealth mode / Ad blocking attempts
+            # Enhanced headers for stealth
-                await context.route("**/*", lambda route: route.continue_())
+            extra_headers = {
+                "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Cache-Control": "max-age=0",
+                "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+                "Sec-Ch-Ua-Mobile": "?0",
+                "Sec-Ch-Ua-Platform": '"Windows"',
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+
+            # Add referer for Amazon
+            if is_amazon:
+                extra_headers["Referer"] = "https://www.google.fr/"
+
+            try:
+                # Prepare context options
+                context_options = {
+                    "viewport": {"width": 1920, "height": 1080},
+                    "user_agent": current_user_agent,
+                    "locale": "fr-FR",
+                    "timezone_id": "Europe/Paris",
+                    "extra_http_headers": extra_headers,
+                    "java_script_enabled": True,
+                    "bypass_csp": True,
+                }
+
+                # Add proxy for Amazon if configured
+                if is_amazon:
+                    proxy_url = _get_amazon_proxy()
+                    if proxy_url:
+                        logger.info(f"Using proxy for Amazon: {proxy_url.split('@')[-1] if '@' in proxy_url else proxy_url}")
+                        context_options["proxy"] = {"server": proxy_url}
+
+                context = await current_browser.new_context(**context_options)
+
+                # Inject stealth script before any page loads (especially for Amazon)
+                if is_amazon:
+                    await context.add_init_script(STEALTH_JS)
+                    logger.debug("Stealth JS injected for Amazon product page")
+
+                # Stealth mode / Ad blocking attempts - don't block images for Amazon
+                if is_amazon:
+                    # For Amazon, only block tracking - keep images to avoid detection
+                    await context.route("**/analytics*", lambda route: route.abort())
+                    await context.route("**/tracking*", lambda route: route.abort())
+                else:
+                    await context.route("**/*", lambda route: route.continue_())
 
                 page = await context.new_page()
 
+                # Add random delay before navigation for Amazon
+                if is_amazon:
+                    await asyncio.sleep(random.uniform(1.0, 3.0))
+                    
+                    # Warmup: Visit home page first
+                    try:
+                        logger.info("Amazon Warmup: Visiting home page...")
+                        await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
+                        await asyncio.sleep(random.uniform(2.0, 4.0))
+                    except Exception as e:
+                        logger.warning(f"Amazon warmup failed: {e}")
+
                 logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
                 response = None
+                amazon_blocked = False
                 try:
                     # First wait for domcontentloaded - this is the minimum we need
                     response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
@@ -214,8 +586,15 @@ class ScraperService:
                         status = response.status
                         logger.info(f"HTTP Status: {status}")
 
-                        # Detect unavailable products by HTTP status
+                        # For Amazon, 503 is usually a block, not product unavailability
-                        if status in [404, 410, 451, 503]:
+                        if is_amazon and status == 503:
+                            logger.warning(f"Amazon returned 503 - likely blocking")
+                            amazon_blocked = True
+                        # Detect unavailable products by HTTP status (non-Amazon)
+                        elif status in [404, 410, 451]:
+                            logger.warning(f"Product unavailable - HTTP {status}")
+                            is_available = False
+                        elif status == 503 and not is_amazon:
                             logger.warning(f"Product unavailable - HTTP {status}")
                             is_available = False
                         elif status >= 400:
@@ -239,8 +618,24 @@ class ScraperService:
                     # Try to take screenshot anyway if page partially loaded
                     pass
 
-                # Wait a bit for dynamic content if needed
+                # Wait a bit for dynamic content if needed - longer for Amazon
-                await page.wait_for_timeout(2000)
+                wait_time = random.uniform(2.0, 4.0) if is_amazon else 2.0
+                await page.wait_for_timeout(int(wait_time * 1000))
+
+                # Check for Amazon blocking in page content
+                if is_amazon:
+                    try:
+                        page_html = await page.content()
+                        if _is_amazon_blocked(page_html):
+                            logger.warning(f"Amazon blocking detected in page content for {url}")
+                            amazon_blocked = True
+                    except Exception as e:
+                        logger.debug(f"Could not check Amazon blocking: {e}")
+
+                # If Amazon blocked us, return None to trigger retry
+                if amazon_blocked:
+                    logger.warning(f"Amazon blocked - will retry with different User-Agent")
+                    return None, "", True  # Return None screenshot to trigger retry
 
                 # Try to close common popups and cookie banners
                 logger.info("Attempting to close popups and cookie banners...")

app/services/search_service.py

@@ -76,6 +76,7 @@ async def search_products(
             "search_url": site.search_url,
             "product_link_selector": site.product_link_selector,
             "requires_js": site.requires_js,
+            "debug_enabled": site.debug_enabled,
         }
         for site in sites
     ]

debug_amazon.py

@@ -0,0 +1,60 @@
+import asyncio
+import logging
+import os
+from playwright.async_api import async_playwright
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
+
+async def debug_amazon():
+    logger.info("Starting Amazon Debug Script")
+    
+    async with async_playwright() as p:
+        try:
+            logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+            browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
+            
+            # Use a very standard, recent User-Agent
+            user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            
+            context = await browser.new_context(
+                user_agent=user_agent,
+                viewport={"width": 1920, "height": 1080},
+                locale="fr-FR",
+                timezone_id="Europe/Paris"
+            )
+            
+            page = await context.new_page()
+            
+            url = "https://www.amazon.fr/s?k=iphone"
+            logger.info(f"Navigating to {url}")
+            
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            
+            if response:
+                status = response.status
+                logger.info(f"Response Status: {status}")
+                
+                content = await page.content()
+                if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content:
+                    logger.error("BLOCK DETECTED: Found blocking message in content")
+                else:
+                    logger.info("No obvious blocking message found")
+                    
+                # Save screenshot
+                await page.screenshot(path="debug_amazon_screenshot.png")
+                logger.info("Screenshot saved to debug_amazon_screenshot.png")
+                
+            else:
+                logger.error("No response received")
+                
+            await browser.close()
+            
+        except Exception as e:
+            logger.error(f"An error occurred: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(debug_amazon())

frontend/src/pages/Admin.jsx

@@ -437,13 +437,43 @@ export default function Admin() {
     // Site functions
     const toggleSiteActive = async (site) => {
         try {
-            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active });
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() });
             fetchAll();
         } catch (error) {
             toast.error('Erreur lors de la mise à jour');
         }
     };
 
+    const toggleSiteDebug = async (site) => {
+        try {
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() });
+            toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé');
+            fetchAll();
+        } catch (error) {
+            toast.error('Erreur lors de la mise à jour du debug');
+        }
+    };
+
+    const fetchSiteDebugFiles = async (domain) => {
+        try {
+            const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            return response.data;
+        } catch (error) {
+            console.error('Error fetching debug files:', error);
+            return [];
+        }
+    };
+
+    const deleteSiteDebugFiles = async (domain) => {
+        if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return;
+        try {
+            const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            toast.success(response.data.message);
+        } catch (error) {
+            toast.error('Erreur lors de la suppression');
+        }
+    };
+
     const resetSitesToDefaults = async () => {
         if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return;
         setResettingSites(true);
@@ -862,11 +892,35 @@ export default function Admin() {
                                         </div>
                                         <Switch checked={site.is_active} onCheckedChange={() => toggleSiteActive(site)} className="ml-2" />
                                     </div>
-                                    {site.category && (
+                                    <div className="flex items-center justify-between">
-                                        <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
+                                        {site.category && (
-                                            {site.category}
+                                            <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
-                                        </span>
+                                                {site.category}
-                                    )}
+                                            </span>
+                                        )}
+                                        <div className="flex items-center gap-2 ml-auto">
+                                            <Button
+                                                variant="ghost"
+                                                size="sm"
+                                                className={cn("h-7 px-2", site.debug_enabled && "text-orange-500")}
+                                                onClick={() => toggleSiteDebug(site)}
+                                                title={site.debug_enabled ? "Désactiver le debug" : "Activer le debug"}
+                                            >
+                                                <Bug className="h-4 w-4" />
+                                            </Button>
+                                            {site.debug_enabled && (
+                                                <Button
+                                                    variant="ghost"
+                                                    size="sm"
+                                                    className="h-7 px-2"
+                                                    onClick={() => deleteSiteDebugFiles(site.domain)}
+                                                    title="Supprimer fichiers debug"
+                                                >
+                                                    <Trash2 className="h-4 w-4" />
+                                                </Button>
+                                            )}
+                                        </div>
+                                    </div>
                                 </div>
                             ))}
                         </div>
@@ -1018,6 +1072,54 @@ export default function Admin() {
                     </CardContent>
                 </Card>
             )}
+
+            {/* Debug Tab */}
+            {activeTab === 'debug' && (
+                <Card>
+                    <CardHeader>
+                        <div className="flex items-center justify-between">
+                            <div>
+                                <CardTitle className="flex items-center gap-2"><Bug className="h-5 w-5" />Debug HTML Dumps</CardTitle>
+                                <CardDescription>Fichiers HTML sauvegardés lors des recherches sans résultats.</CardDescription>
+                            </div>
+                            <Button variant="outline" size="sm" onClick={fetchDebugFiles} disabled={loadingDebugFiles}>
+                                <RefreshCw className={cn("h-4 w-4 mr-2", loadingDebugFiles && "animate-spin")} />
+                                Actualiser
+                            </Button>
+                        </div>
+                    </CardHeader>
+                    <CardContent>
+                        {loadingDebugFiles ? (
+                            <div className="text-center py-8 text-muted-foreground">Chargement...</div>
+                        ) : debugFiles.length === 0 ? (
+                            <div className="text-center py-8 text-muted-foreground">
+                                Aucun fichier de debug disponible.
+                            </div>
+                        ) : (
+                            <div className="space-y-2">
+                                {debugFiles.map(file => (
+                                    <div key={file.filename} className="flex items-center justify-between p-3 rounded-lg border bg-card hover:bg-muted/50 transition-colors">
+                                        <div className="flex-1 min-w-0">
+                                            <p className="font-medium text-sm truncate">{file.filename}</p>
+                                            <p className="text-xs text-muted-foreground">
+                                                {(file.size / 1024).toFixed(1)} KB • {file.created_at_formatted}
+                                            </p>
+                                        </div>
+                                        <div className="flex items-center gap-1 ml-4">
+                                            <Button variant="ghost" size="icon" className="h-8 w-8" onClick={() => downloadDebugFile(file.filename)} title="Télécharger">
+                                                <Download className="h-4 w-4" />
+                                            </Button>
+                                            <Button variant="ghost" size="icon" className="h-8 w-8 text-destructive hover:text-destructive" onClick={() => deleteDebugFile(file.filename)} title="Supprimer">
+                                                <Trash2 className="h-4 w-4" />
+                                            </Button>
+                                        </div>
+                                    </div>
+                                ))}
+                            </div>
+                        )}
+                    </CardContent>
+                </Card>
+            )}
         </div >
     );
 }