feat: implement direct e-commerce search service with Playwright/Browserless and add supporting admin page and debug routes.

app/main.py

@@ -13,7 +13,7 @@ from sqlalchemy import text
 
 from app.database import SessionLocal, engine
 from app.limiter import limiter
-from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings
+from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug
 from app.services.scheduler_service import scheduled_refresh, scheduler
 from app.services import auth_service, search_service
 
@@ -163,7 +163,7 @@ os.makedirs("screenshots", exist_ok=True)
 app.mount("/screenshots", StaticFiles(directory="screenshots"), name="screenshots")
 
 # Routers
-for router in [notifications.router, items.router, settings.router, jobs.router, openrouter.router]:
+for router in [notifications.router, items.router, settings.router, jobs.router, openrouter.router, debug.router]:
     app.include_router(router, prefix="/api")
 
 # Search routers (already have /api prefix)

app/routers/debug.py

@@ -0,0 +1,91 @@
+import os
+import logging
+from datetime import datetime
+from typing import List
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from app.services import auth_service
+
+router = APIRouter(
+    prefix="/debug",
+    tags=["debug"],
+    dependencies=[Depends(auth_service.get_current_active_admin)],
+)
+
+logger = logging.getLogger(__name__)
+
+DEBUG_DIR = "debug_dumps"
+
+class DebugFile(BaseModel):
+    filename: str
+    size: int
+    created_at: float
+    created_at_formatted: str
+
+@router.get("/dumps", response_model=List[DebugFile])
+def list_debug_dumps():
+    """List all debug dump files"""
+    if not os.path.exists(DEBUG_DIR):
+        return []
+    
+    files = []
+    try:
+        for filename in os.listdir(DEBUG_DIR):
+            filepath = os.path.join(DEBUG_DIR, filename)
+            if os.path.isfile(filepath):
+                stat = os.stat(filepath)
+                files.append(DebugFile(
+                    filename=filename,
+                    size=stat.st_size,
+                    created_at=stat.st_ctime,
+                    created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S")
+                ))
+        
+        # Sort by creation time (newest first)
+        files.sort(key=lambda x: x.created_at, reverse=True)
+        return files
+    except Exception as e:
+        logger.error(f"Error listing debug dumps: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/dumps/{filename}")
+def download_debug_dump(filename: str):
+    """Download a specific debug dump file"""
+    if not os.path.exists(DEBUG_DIR):
+        raise HTTPException(status_code=404, detail="Debug directory not found")
+    
+    filepath = os.path.join(DEBUG_DIR, filename)
+    
+    # Security check: prevent directory traversal
+    if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)):
+        raise HTTPException(status_code=403, detail="Access denied")
+        
+    if not os.path.isfile(filepath):
+        raise HTTPException(status_code=404, detail="File not found")
+        
+    return FileResponse(filepath, filename=filename)
+
+@router.delete("/dumps/{filename}")
+def delete_debug_dump(filename: str):
+    """Delete a specific debug dump file"""
+    if not os.path.exists(DEBUG_DIR):
+        raise HTTPException(status_code=404, detail="Debug directory not found")
+    
+    filepath = os.path.join(DEBUG_DIR, filename)
+    
+    # Security check: prevent directory traversal
+    if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)):
+        raise HTTPException(status_code=403, detail="Access denied")
+        
+    if not os.path.isfile(filepath):
+        raise HTTPException(status_code=404, detail="File not found")
+    
+    try:
+        os.remove(filepath)
+        return {"message": "File deleted"}
+    except Exception as e:
+        logger.error(f"Error deleting debug dump {filename}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

app/services/direct_search_service.py

@@ -21,10 +21,25 @@ MIN_TITLE_LENGTH = 3
 MIN_LINK_TEXT_LENGTH = 5
 USER_AGENT = (
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
 )
 
 
+def _dump_debug_html(html: str, domain: str, query: str):
+    """Sauvegarde le HTML pour débogage"""
+    try:
+        debug_dir = "debug_dumps"
+        if not os.path.exists(debug_dir):
+            os.makedirs(debug_dir)
+            
+        filename = f"{debug_dir}/{domain}_{query}_{int(asyncio.get_event_loop().time())}.html"
+        with open(filename, "w", encoding="utf-8") as f:
+            f.write(html)
+        logger.info(f"HTML dump saved to {filename}")
+    except Exception as e:
+        logger.error(f"Failed to dump HTML: {e}")
+
+
 class SearchResult:
     """Résultat de recherche"""
 
@@ -280,6 +295,7 @@ DEFAULT_SITE_CONFIGS = {
     },
 }
 
+
 # Sélecteurs communs pour les bannières de cookies
 # Ordonnés du plus spécifique au plus générique pour une meilleure détection
 COOKIE_ACCEPT_SELECTORS = [
@@ -571,6 +587,12 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
         # Extraire le HTML
         html_content = await page.content()
 
+        # Check for CAPTCHA
+        if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
+            logger.warning(f"CAPTCHA detected on {domain}!")
+            _dump_debug_html(html_content, domain, query)
+            return []
+
         # Parser le HTML avec BeautifulSoup
         soup = BeautifulSoup(html_content, "lxml")
         results = []
@@ -617,15 +639,18 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
             # Vérifier que c'est bien un lien vers ce domaine
             url_domain = _extract_domain(href)
             if domain not in url_domain and url_domain not in domain:
+                # logger.debug(f"Ignored external link: {href}")
                 continue
 
             # Filtrer les liens non-produits
             if _is_non_product_url(href):
+                # logger.debug(f"Ignored non-product link: {href}")
                 continue
 
             # Extraire le titre
             title = _extract_title(link)
             if not title or len(title) < MIN_TITLE_LENGTH:
+                # logger.debug(f"Ignored link with no title: {href}")
                 continue
 
             results.append(SearchResult(
@@ -641,6 +666,11 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 break
 
         logger.info(f"{domain}: {len(results)} résultats extraits")
+        
+        if len(results) == 0:
+            logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
+            _dump_debug_html(html_content, domain, query)
+            
         return results
 
     except Exception as e:

frontend/src/pages/Admin.jsx

@@ -7,7 +7,7 @@ import {
     Settings as SettingsIcon, Cpu, Clock, Bell, Globe,
     ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2,
     Sparkles, Eye, Code, Brain, DollarSign, Filter,
-    TrendingDown, CheckCircle2, Package
+    TrendingDown, CheckCircle2, Package, Bug, Download
 } from 'lucide-react';
 import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
 import { Label } from '@/components/ui/label';
@@ -87,7 +87,56 @@ export default function Admin() {
         { id: 'code', name: 'Code', description: 'Programmation' },
         { id: 'reasoning', name: 'Raisonnement', description: 'Raisonnement avancé' },
         { id: 'free', name: 'Gratuit', description: 'Modèles gratuits' }
-    ]);
+    const [debugFiles, setDebugFiles] = useState([]);
+    const [loadingDebugFiles, setLoadingDebugFiles] = useState(false);
+
+    useEffect(() => {
+        if (activeTab === 'debug') {
+            fetchDebugFiles();
+        }
+    }, [activeTab]);
+
+    const fetchDebugFiles = async () => {
+        setLoadingDebugFiles(true);
+        try {
+            const response = await axios.get(`${API_URL}/debug/dumps`, { headers: getAuthHeaders() });
+            setDebugFiles(response.data);
+        } catch (error) {
+            console.error('Error fetching debug files:', error);
+            toast.error('Erreur lors du chargement des fichiers');
+        } finally {
+            setLoadingDebugFiles(false);
+        }
+    };
+
+    const downloadDebugFile = async (filename) => {
+        try {
+            const response = await axios.get(`${API_URL}/debug/dumps/${filename}`, {
+                headers: getAuthHeaders(),
+                responseType: 'blob'
+            });
+            const url = window.URL.createObjectURL(new Blob([response.data]));
+            const link = document.createElement('a');
+            link.href = url;
+            link.setAttribute('download', filename);
+            document.body.appendChild(link);
+            link.click();
+            link.remove();
+        } catch (error) {
+            toast.error('Erreur lors du téléchargement');
+        }
+    };
+
+    const deleteDebugFile = async (filename) => {
+        if (!confirm(`Supprimer ${filename} ?`)) return;
+        try {
+            await axios.delete(`${API_URL}/debug/dumps/${filename}`, { headers: getAuthHeaders() });
+            toast.success('Fichier supprimé');
+            fetchDebugFiles();
+        } catch (error) {
+            toast.error('Erreur lors de la suppression');
+        }
+    };
 
     useEffect(() => {
         if (!isAdmin) {
@@ -415,6 +464,7 @@ export default function Admin() {
         { id: 'sites', label: 'Sites', icon: Globe },
         { id: 'jobs', label: 'Jobs', icon: Clock },
         { id: 'notifications', label: 'Notifications', icon: Bell },
+        { id: 'debug', label: 'Debug', icon: Bug },
     ];
 
     return (