Merge pull request #68 from R0m1k3/claude/fix-amazon-search-error-01Gnft8Nyjh9KkpKuavHaaPc

Claude/fix amazon search error 01 gnft8 nyjh9 kkp kuav haa pc

app/models.py

@@ -94,6 +94,7 @@ class SearchSite(Base):
     is_active: bool = Column(Boolean, default=True)  # type: ignore
     priority: int = Column(Integer, default=0)  # type: ignore  # Ordre d'affichage
     requires_js: bool = Column(Boolean, default=False)  # type: ignore  # Force Browserless si True
+    debug_enabled: bool = Column(Boolean, default=False)  # type: ignore  # Activer le dump HTML pour ce site
     price_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour le prix
     search_url: str | None = Column(String, nullable=True)  # type: ignore  # URL de recherche avec {query} placeholder
     product_link_selector: str | None = Column(String, nullable=True)  # type: ignore  # Sélecteur CSS pour les liens produits

app/routers/debug.py

@@ -75,19 +75,82 @@ def delete_debug_dump(filename: str):
     """Delete a specific debug dump file"""
     if not os.path.exists(DEBUG_DIR):
         raise HTTPException(status_code=404, detail="Debug directory not found")
-    
+
     filepath = os.path.join(DEBUG_DIR, filename)
-    
+
     # Security check: prevent directory traversal
     if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)):
         raise HTTPException(status_code=403, detail="Access denied")
-        
+
     if not os.path.isfile(filepath):
         raise HTTPException(status_code=404, detail="File not found")
-    
+
     try:
         os.remove(filepath)
         return {"message": "File deleted"}
     except Exception as e:
         logger.error(f"Error deleting debug dump {filename}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/dumps/site/{domain}", response_model=List[DebugFile])
+def list_debug_dumps_by_site(domain: str):
+    """List debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return []
+
+    # Clean domain for comparison (remove www. if present)
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    files = []
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            # Check if filename starts with the domain
+            filename_lower = filename.lower()
+            # Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    stat = os.stat(filepath)
+                    files.append(DebugFile(
+                        filename=filename,
+                        size=stat.st_size,
+                        created_at=stat.st_ctime,
+                        created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S")
+                    ))
+
+        # Sort by creation time (newest first)
+        files.sort(key=lambda x: x.created_at, reverse=True)
+        return files
+    except Exception as e:
+        logger.error(f"Error listing debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/dumps/site/{domain}")
+def delete_debug_dumps_by_site(domain: str):
+    """Delete all debug dump files for a specific site domain"""
+    if not os.path.exists(DEBUG_DIR):
+        return {"message": "No files found", "deleted": 0}
+
+    # Clean domain for comparison
+    clean_domain = domain.lower().strip()
+    if clean_domain.startswith("www."):
+        clean_domain = clean_domain[4:]
+
+    deleted_count = 0
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            filename_lower = filename.lower()
+            if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
+                filepath = os.path.join(DEBUG_DIR, filename)
+                if os.path.isfile(filepath):
+                    os.remove(filepath)
+                    deleted_count += 1
+
+        return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count}
+    except Exception as e:
+        logger.error(f"Error deleting debug dumps for site {domain}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

app/routers/search_sites.py

@@ -41,17 +41,17 @@ async def update_site(
 ):
     """
     Met à jour un site de recherche.
-    Permet uniquement de modifier is_active et priority.
+    Permet uniquement de modifier is_active, priority et debug_enabled.
     """
     # Limiter les champs modifiables
-    allowed_fields = {"is_active", "priority"}
+    allowed_fields = {"is_active", "priority", "debug_enabled"}
     update_data = site.model_dump(exclude_unset=True)
     filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields}
 
     if not filtered_data:
         raise HTTPException(
             status_code=400,
-            detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés"
+            detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés"
         )
 
     updated = search_service.update_site(db, site_id, filtered_data)

app/schemas.py

@@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel):
     is_active: bool = True
     priority: int = 0
     requires_js: bool = False
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None  # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query}
     product_link_selector: str | None = None  # Sélecteur CSS pour les liens produits
@@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel):
     is_active: bool | None = None
     priority: int | None = None
     requires_js: bool | None = None
+    debug_enabled: bool | None = None  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None
@@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel):
     is_active: bool
     priority: int
     requires_js: bool
+    debug_enabled: bool = False  # Activer le dump HTML pour ce site
     price_selector: str | None = None
     search_url: str | None = None
     product_link_selector: str | None = None

app/services/direct_search_service.py

@@ -109,6 +109,106 @@ async def _simulate_human_behavior(page):
 USER_AGENT = USER_AGENT_POOL[0]
 
 
+# Stealth JavaScript to inject before page load to hide automation
+STEALTH_JS = """
+// Override the navigator.webdriver property
+Object.defineProperty(navigator, 'webdriver', {
+    get: () => undefined,
+    configurable: true
+});
+
+// Override navigator.plugins to have some plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+        ];
+        plugins.item = (i) => plugins[i];
+        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.refresh = () => {};
+        return plugins;
+    },
+    configurable: true
+});
+
+// Override navigator.languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['fr-FR', 'fr', 'en-US', 'en'],
+    configurable: true
+});
+
+// Override permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) => (
+    parameters.name === 'notifications' ?
+        Promise.resolve({ state: Notification.permission }) :
+        originalQuery(parameters)
+);
+
+// Hide automation-related properties
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+// Override chrome object
+window.chrome = {
+    runtime: {},
+    loadTimes: function() {},
+    csi: function() {},
+    app: {}
+};
+
+// Fake WebGL vendor and renderer
+const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+        if (args[0] === 37445) {
+            return 'Intel Inc.';
+        }
+        if (args[0] === 37446) {
+            return 'Intel Iris OpenGL Engine';
+        }
+        return Reflect.apply(target, ctx, args);
+    }
+};
+
+try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+        const getParameter = gl.getParameter.bind(gl);
+        gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
+    }
+} catch(e) {}
+
+// Override connection rtt (reduces fingerprinting)
+Object.defineProperty(navigator, 'connection', {
+    get: () => ({
+        effectiveType: '4g',
+        rtt: 50,
+        downlink: 10,
+        saveData: false
+    }),
+    configurable: true
+});
+
+// Override hardware concurrency
+Object.defineProperty(navigator, 'hardwareConcurrency', {
+    get: () => 8,
+    configurable: true
+});
+
+// Override device memory
+Object.defineProperty(navigator, 'deviceMemory', {
+    get: () => 8,
+    configurable: true
+});
+
+console.log('Stealth mode activated');
+"""
+
+
 # Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py)
 DEBUG_DUMPS_DIR = "/app/debug_dumps"
 
@@ -631,6 +731,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
     # Nettoyer le domaine (enlever protocole, www, slash final)
     domain = _clean_domain(raw_domain)
 
+    # Check if debug is enabled for this site
+    debug_enabled = site.get("debug_enabled", False)
+
     search_url = site.get("search_url") or _get_default_search_url(domain)
     product_selector = site.get("product_link_selector") or _get_default_product_selector(domain)
     wait_selector = _get_default_wait_selector(domain)
@@ -692,8 +795,16 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 locale="fr-FR",
                 timezone_id="Europe/Paris",
                 extra_http_headers=extra_headers,
+                # Additional stealth options
+                java_script_enabled=True,
+                bypass_csp=True,  # Bypass Content Security Policy for script injection
             )
 
+            # Inject stealth script before any page loads (especially for Amazon)
+            if is_amazon:
+                await context.add_init_script(STEALTH_JS)
+                logger.debug("Stealth JS injected for Amazon")
+
             # For Amazon, don't block images - it may trigger detection
             if not is_amazon:
                 await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
@@ -733,7 +844,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             # Check for Amazon blocking (503, "Toutes nos excuses", etc.)
             if is_amazon and _is_amazon_blocked(html_content):
                 logger.warning(f"Amazon blocked request (attempt {attempt + 1}/{max_retries}) for '{query}'")
-                _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
+                # Always dump for Amazon blocking (critical error)
+                if debug_enabled or is_amazon:
+                    _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
                 # Close resources before retry
                 try:
                     if page:
@@ -747,7 +860,9 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             # Check for CAPTCHA
             if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
                 logger.warning(f"CAPTCHA detected on {domain}!")
-                _dump_debug_html(html_content, domain, query)
+                # Always dump for CAPTCHA (critical error)
+                if debug_enabled or is_amazon:
+                    _dump_debug_html(html_content, domain, f"{query}_captcha")
                 if is_amazon and attempt < max_retries - 1:
                     try:
                         if page:
@@ -838,8 +953,11 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 return results
 
             # No results found - for Amazon this might be temporary blocking
-            logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
-            _dump_debug_html(html_content, domain, query)
+            if debug_enabled:
+                logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
+                _dump_debug_html(html_content, domain, f"{query}_no_results")
+            else:
+                logger.warning(f"{domain}: 0 results found (debug disabled, no HTML dump)")
 
             # For Amazon, if no results and we have retries left, try again
             if is_amazon and attempt < max_retries - 1:

app/services/scraper_service.py

@@ -41,6 +41,105 @@ AMAZON_BLOCK_INDICATORS = [
     "automated access",
 ]
 
+# Stealth JavaScript to inject before page load to hide automation
+STEALTH_JS = """
+// Override the navigator.webdriver property
+Object.defineProperty(navigator, 'webdriver', {
+    get: () => undefined,
+    configurable: true
+});
+
+// Override navigator.plugins to have some plugins
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+            { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+            { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
+        ];
+        plugins.item = (i) => plugins[i];
+        plugins.namedItem = (name) => plugins.find(p => p.name === name);
+        plugins.refresh = () => {};
+        return plugins;
+    },
+    configurable: true
+});
+
+// Override navigator.languages
+Object.defineProperty(navigator, 'languages', {
+    get: () => ['fr-FR', 'fr', 'en-US', 'en'],
+    configurable: true
+});
+
+// Override permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) => (
+    parameters.name === 'notifications' ?
+        Promise.resolve({ state: Notification.permission }) :
+        originalQuery(parameters)
+);
+
+// Hide automation-related properties
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+// Override chrome object
+window.chrome = {
+    runtime: {},
+    loadTimes: function() {},
+    csi: function() {},
+    app: {}
+};
+
+// Fake WebGL vendor and renderer
+const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+        if (args[0] === 37445) {
+            return 'Intel Inc.';
+        }
+        if (args[0] === 37446) {
+            return 'Intel Iris OpenGL Engine';
+        }
+        return Reflect.apply(target, ctx, args);
+    }
+};
+
+try {
+    const canvas = document.createElement('canvas');
+    const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (gl) {
+        const getParameter = gl.getParameter.bind(gl);
+        gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
+    }
+} catch(e) {}
+
+// Override connection rtt (reduces fingerprinting)
+Object.defineProperty(navigator, 'connection', {
+    get: () => ({
+        effectiveType: '4g',
+        rtt: 50,
+        downlink: 10,
+        saveData: false
+    }),
+    configurable: true
+});
+
+// Override hardware concurrency
+Object.defineProperty(navigator, 'hardwareConcurrency', {
+    get: () => 8,
+    configurable: true
+});
+
+// Override device memory
+Object.defineProperty(navigator, 'deviceMemory', {
+    get: () => 8,
+    configurable: true
+});
+
+console.log('Stealth mode activated');
+"""
+
 
 def _get_random_user_agent() -> str:
     """Get a random User-Agent from the pool"""
@@ -269,10 +368,23 @@ class ScraperService:
                     locale="fr-FR",
                     timezone_id="Europe/Paris",
                     extra_http_headers=extra_headers,
+                    # Additional stealth options
+                    java_script_enabled=True,
+                    bypass_csp=True,  # Bypass Content Security Policy for script injection
                 )
 
-                # Stealth mode / Ad blocking attempts
-                await context.route("**/*", lambda route: route.continue_())
+                # Inject stealth script before any page loads (especially for Amazon)
+                if is_amazon:
+                    await context.add_init_script(STEALTH_JS)
+                    logger.debug("Stealth JS injected for Amazon product page")
+
+                # Stealth mode / Ad blocking attempts - don't block images for Amazon
+                if is_amazon:
+                    # For Amazon, only block tracking - keep images to avoid detection
+                    await context.route("**/analytics*", lambda route: route.abort())
+                    await context.route("**/tracking*", lambda route: route.abort())
+                else:
+                    await context.route("**/*", lambda route: route.continue_())
 
                 page = await context.new_page()
 

app/services/search_service.py

@@ -76,6 +76,7 @@ async def search_products(
             "search_url": site.search_url,
             "product_link_selector": site.product_link_selector,
             "requires_js": site.requires_js,
+            "debug_enabled": site.debug_enabled,
         }
         for site in sites
     ]

frontend/src/pages/Admin.jsx

@@ -437,13 +437,51 @@ export default function Admin() {
     // Site functions
     const toggleSiteActive = async (site) => {
         try {
-            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active });
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() });
             fetchAll();
         } catch (error) {
             toast.error('Erreur lors de la mise à jour');
         }
     };
 
+    const toggleSiteDebug = async (site) => {
+        try {
+            await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() });
+            toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé');
+            fetchAll();
+        } catch (error) {
+            toast.error('Erreur lors de la mise à jour du debug');
+        }
+    };
+
+    const fetchSiteDebugFiles = async (domain) => {
+        try {
+            const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            return response.data;
+        } catch (error) {
+            console.error('Error fetching debug files:', error);
+            return [];
+        }
+    };
+
+    const downloadDebugFile = async (filename) => {
+        try {
+            window.open(`${API_URL}/debug/dumps/${filename}`, '_blank');
+        } catch (error) {
+            toast.error('Erreur lors du téléchargement');
+        }
+    };
+
+    const deleteSiteDebugFiles = async (domain) => {
+        if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return;
+        try {
+            const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
+            toast.success(response.data.message);
+        } catch (error) {
+            toast.error('Erreur lors de la suppression');
+        }
+    };
+
     const resetSitesToDefaults = async () => {
         if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return;
         setResettingSites(true);
@@ -862,11 +900,35 @@ export default function Admin() {
                                         </div>
                                         <Switch checked={site.is_active} onCheckedChange={() => toggleSiteActive(site)} className="ml-2" />
                                     </div>
-                                    {site.category && (
-                                        <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
-                                            {site.category}
-                                        </span>
-                                    )}
+                                    <div className="flex items-center justify-between">
+                                        {site.category && (
+                                            <span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
+                                                {site.category}
+                                            </span>
+                                        )}
+                                        <div className="flex items-center gap-2 ml-auto">
+                                            <Button
+                                                variant="ghost"
+                                                size="sm"
+                                                className={cn("h-7 px-2", site.debug_enabled && "text-orange-500")}
+                                                onClick={() => toggleSiteDebug(site)}
+                                                title={site.debug_enabled ? "Désactiver le debug" : "Activer le debug"}
+                                            >
+                                                <Bug className="h-4 w-4" />
+                                            </Button>
+                                            {site.debug_enabled && (
+                                                <Button
+                                                    variant="ghost"
+                                                    size="sm"
+                                                    className="h-7 px-2"
+                                                    onClick={() => deleteSiteDebugFiles(site.domain)}
+                                                    title="Supprimer fichiers debug"
+                                                >
+                                                    <Trash2 className="h-4 w-4" />
+                                                </Button>
+                                            )}
+                                        </div>
+                                    </div>
                                 </div>
                             ))}
                         </div>