From e45290350cc52ced93089ebe2867ae64a6a7b276 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Mon, 24 Nov 2025 07:18:12 +0100 Subject: [PATCH] feat: implement direct e-commerce search service with Playwright/Browserless and add supporting admin page and debug routes. --- app/main.py | 4 +- app/routers/debug.py | 91 +++++++++++++++++++++++++++ app/services/direct_search_service.py | 32 +++++++++- frontend/src/pages/Admin.jsx | 54 +++++++++++++++- 4 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 app/routers/debug.py diff --git a/app/main.py b/app/main.py index 824aada..a760996 100644 --- a/app/main.py +++ b/app/main.py @@ -13,7 +13,7 @@ from sqlalchemy import text from app.database import SessionLocal, engine from app.limiter import limiter -from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings +from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug from app.services.scheduler_service import scheduled_refresh, scheduler from app.services import auth_service, search_service @@ -163,7 +163,7 @@ os.makedirs("screenshots", exist_ok=True) app.mount("/screenshots", StaticFiles(directory="screenshots"), name="screenshots") # Routers -for router in [notifications.router, items.router, settings.router, jobs.router, openrouter.router]: +for router in [notifications.router, items.router, settings.router, jobs.router, openrouter.router, debug.router]: app.include_router(router, prefix="/api") # Search routers (already have /api prefix) diff --git a/app/routers/debug.py b/app/routers/debug.py new file mode 100644 index 0000000..9ae797e --- /dev/null +++ b/app/routers/debug.py @@ -0,0 +1,91 @@ +import os +import logging +from datetime import datetime +from typing import List + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from app.services import auth_service + +router = APIRouter( + prefix="/debug", + tags=["debug"], + dependencies=[Depends(auth_service.get_current_active_admin)], +) + +logger = logging.getLogger(__name__) + +DEBUG_DIR = "debug_dumps" + +class DebugFile(BaseModel): + filename: str + size: int + created_at: float + created_at_formatted: str + +@router.get("/dumps", response_model=List[DebugFile]) +def list_debug_dumps(): + """List all debug dump files""" + if not os.path.exists(DEBUG_DIR): + return [] + + files = [] + try: + for filename in os.listdir(DEBUG_DIR): + filepath = os.path.join(DEBUG_DIR, filename) + if os.path.isfile(filepath): + stat = os.stat(filepath) + files.append(DebugFile( + filename=filename, + size=stat.st_size, + created_at=stat.st_ctime, + created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S") + )) + + # Sort by creation time (newest first) + files.sort(key=lambda x: x.created_at, reverse=True) + return files + except Exception as e: + logger.error(f"Error listing debug dumps: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@router.get("/dumps/{filename}") +def download_debug_dump(filename: str): + """Download a specific debug dump file""" + if not os.path.exists(DEBUG_DIR): + raise HTTPException(status_code=404, detail="Debug directory not found") + + filepath = os.path.join(DEBUG_DIR, filename) + + # Security check: prevent directory traversal + if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)): + raise HTTPException(status_code=403, detail="Access denied") + + if not os.path.isfile(filepath): + raise HTTPException(status_code=404, detail="File not found") + + return FileResponse(filepath, filename=filename) + +@router.delete("/dumps/{filename}") +def delete_debug_dump(filename: str): + """Delete a specific debug dump file""" + if not os.path.exists(DEBUG_DIR): + raise HTTPException(status_code=404, detail="Debug directory not found") + + filepath = os.path.join(DEBUG_DIR, filename) + + # Security check: prevent directory traversal + if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)): + raise HTTPException(status_code=403, detail="Access denied") + + if not os.path.isfile(filepath): + raise HTTPException(status_code=404, detail="File not found") + + try: + os.remove(filepath) + return {"message": "File deleted"} + except Exception as e: + logger.error(f"Error deleting debug dump {filename}: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py index 73e3f90..4abc547 100644 --- a/app/services/direct_search_service.py +++ b/app/services/direct_search_service.py @@ -21,10 +21,25 @@ MIN_TITLE_LENGTH = 3 MIN_LINK_TEXT_LENGTH = 5 USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" ) +def _dump_debug_html(html: str, domain: str, query: str): + """Sauvegarde le HTML pour débogage""" + try: + debug_dir = "debug_dumps" + if not os.path.exists(debug_dir): + os.makedirs(debug_dir) + + filename = f"{debug_dir}/{domain}_{query}_{int(asyncio.get_event_loop().time())}.html" + with open(filename, "w", encoding="utf-8") as f: + f.write(html) + logger.info(f"HTML dump saved to {filename}") + except Exception as e: + logger.error(f"Failed to dump HTML: {e}") + + class SearchResult: """Résultat de recherche""" @@ -280,6 +295,7 @@ DEFAULT_SITE_CONFIGS = { }, } + # Sélecteurs communs pour les bannières de cookies # Ordonnés du plus spécifique au plus générique pour une meilleure détection COOKIE_ACCEPT_SELECTORS = [ @@ -571,6 +587,12 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Extraire le HTML html_content = await page.content() + # Check for CAPTCHA + if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content: + logger.warning(f"CAPTCHA detected on {domain}!") + _dump_debug_html(html_content, domain, query) + return [] + # Parser le HTML avec BeautifulSoup soup = BeautifulSoup(html_content, "lxml") results = [] @@ -617,15 +639,18 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Vérifier que c'est bien un lien vers ce domaine url_domain = _extract_domain(href) if domain not in url_domain and url_domain not in domain: + # logger.debug(f"Ignored external link: {href}") continue # Filtrer les liens non-produits if _is_non_product_url(href): + # logger.debug(f"Ignored non-product link: {href}") continue # Extraire le titre title = _extract_title(link) if not title or len(title) < MIN_TITLE_LENGTH: + # logger.debug(f"Ignored link with no title: {href}") continue results.append(SearchResult( @@ -641,6 +666,11 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 break logger.info(f"{domain}: {len(results)} résultats extraits") + + if len(results) == 0: + logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.") + _dump_debug_html(html_content, domain, query) + return results except Exception as e: diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx index b0aec90..a8292b1 100644 --- a/frontend/src/pages/Admin.jsx +++ b/frontend/src/pages/Admin.jsx @@ -7,7 +7,7 @@ import { Settings as SettingsIcon, Cpu, Clock, Bell, Globe, ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2, Sparkles, Eye, Code, Brain, DollarSign, Filter, - TrendingDown, CheckCircle2, Package + TrendingDown, CheckCircle2, Package, Bug, Download } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'; import { Label } from '@/components/ui/label'; @@ -87,7 +87,56 @@ export default function Admin() { { id: 'code', name: 'Code', description: 'Programmation' }, { id: 'reasoning', name: 'Raisonnement', description: 'Raisonnement avancé' }, { id: 'free', name: 'Gratuit', description: 'Modèles gratuits' } - ]); + const [debugFiles, setDebugFiles] = useState([]); + const [loadingDebugFiles, setLoadingDebugFiles] = useState(false); + + useEffect(() => { + if (activeTab === 'debug') { + fetchDebugFiles(); + } + }, [activeTab]); + + const fetchDebugFiles = async () => { + setLoadingDebugFiles(true); + try { + const response = await axios.get(`${API_URL}/debug/dumps`, { headers: getAuthHeaders() }); + setDebugFiles(response.data); + } catch (error) { + console.error('Error fetching debug files:', error); + toast.error('Erreur lors du chargement des fichiers'); + } finally { + setLoadingDebugFiles(false); + } + }; + + const downloadDebugFile = async (filename) => { + try { + const response = await axios.get(`${API_URL}/debug/dumps/${filename}`, { + headers: getAuthHeaders(), + responseType: 'blob' + }); + const url = window.URL.createObjectURL(new Blob([response.data])); + const link = document.createElement('a'); + link.href = url; + link.setAttribute('download', filename); + document.body.appendChild(link); + link.click(); + link.remove(); + } catch (error) { + toast.error('Erreur lors du téléchargement'); + } + }; + + const deleteDebugFile = async (filename) => { + if (!confirm(`Supprimer ${filename} ?`)) return; + try { + await axios.delete(`${API_URL}/debug/dumps/${filename}`, { headers: getAuthHeaders() }); + toast.success('Fichier supprimé'); + fetchDebugFiles(); + } catch (error) { + toast.error('Erreur lors de la suppression'); + } + }; useEffect(() => { if (!isAdmin) { @@ -415,6 +464,7 @@ export default function Admin() { { id: 'sites', label: 'Sites', icon: Globe }, { id: 'jobs', label: 'Jobs', icon: Clock }, { id: 'notifications', label: 'Notifications', icon: Bell }, + { id: 'debug', label: 'Debug', icon: Bug }, ]; return (