From e62117f9b9bff1eb3d64a2d2b649ef288c629878 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Nov 2025 08:01:39 +0000 Subject: [PATCH] feat: Add per-site debug toggle for HTML dump control - Add debug_enabled field to SearchSite model - Update schemas and API routes to allow debug toggle - Add debug toggle button on site cards in Admin UI - Only dump HTML when debug is enabled for the site (except for critical errors like Amazon blocking) - Add API endpoints to list/delete debug files per site domain - Add button to clear debug files for individual sites --- app/models.py | 1 + app/routers/debug.py | 71 +++++++++++++++++++++++-- app/routers/search_sites.py | 6 +-- app/schemas.py | 3 ++ app/services/direct_search_service.py | 18 +++++-- app/services/search_service.py | 1 + frontend/src/pages/Admin.jsx | 74 ++++++++++++++++++++++++--- 7 files changed, 157 insertions(+), 17 deletions(-) diff --git a/app/models.py b/app/models.py index cbee95e..d5d61e3 100644 --- a/app/models.py +++ b/app/models.py @@ -94,6 +94,7 @@ class SearchSite(Base): is_active: bool = Column(Boolean, default=True) # type: ignore priority: int = Column(Integer, default=0) # type: ignore # Ordre d'affichage requires_js: bool = Column(Boolean, default=False) # type: ignore # Force Browserless si True + debug_enabled: bool = Column(Boolean, default=False) # type: ignore # Activer le dump HTML pour ce site price_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour le prix search_url: str | None = Column(String, nullable=True) # type: ignore # URL de recherche avec {query} placeholder product_link_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour les liens produits diff --git a/app/routers/debug.py b/app/routers/debug.py index 54a9b25..4714a02 100644 --- a/app/routers/debug.py +++ b/app/routers/debug.py @@ -75,19 +75,82 @@ def delete_debug_dump(filename: str): """Delete a specific debug dump file""" if not os.path.exists(DEBUG_DIR): raise HTTPException(status_code=404, detail="Debug directory not found") - + filepath = os.path.join(DEBUG_DIR, filename) - + # Security check: prevent directory traversal if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)): raise HTTPException(status_code=403, detail="Access denied") - + if not os.path.isfile(filepath): raise HTTPException(status_code=404, detail="File not found") - + try: os.remove(filepath) return {"message": "File deleted"} except Exception as e: logger.error(f"Error deleting debug dump {filename}: {e}") raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/dumps/site/{domain}", response_model=List[DebugFile]) +def list_debug_dumps_by_site(domain: str): + """List debug dump files for a specific site domain""" + if not os.path.exists(DEBUG_DIR): + return [] + + # Clean domain for comparison (remove www. if present) + clean_domain = domain.lower().strip() + if clean_domain.startswith("www."): + clean_domain = clean_domain[4:] + + files = [] + try: + for filename in os.listdir(DEBUG_DIR): + # Check if filename starts with the domain + filename_lower = filename.lower() + # Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html + if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")): + filepath = os.path.join(DEBUG_DIR, filename) + if os.path.isfile(filepath): + stat = os.stat(filepath) + files.append(DebugFile( + filename=filename, + size=stat.st_size, + created_at=stat.st_ctime, + created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S") + )) + + # Sort by creation time (newest first) + files.sort(key=lambda x: x.created_at, reverse=True) + return files + except Exception as e: + logger.error(f"Error listing debug dumps for site {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.delete("/dumps/site/{domain}") +def delete_debug_dumps_by_site(domain: str): + """Delete all debug dump files for a specific site domain""" + if not os.path.exists(DEBUG_DIR): + return {"message": "No files found", "deleted": 0} + + # Clean domain for comparison + clean_domain = domain.lower().strip() + if clean_domain.startswith("www."): + clean_domain = clean_domain[4:] + + deleted_count = 0 + try: + for filename in os.listdir(DEBUG_DIR): + filename_lower = filename.lower() + if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")): + filepath = os.path.join(DEBUG_DIR, filename) + if os.path.isfile(filepath): + os.remove(filepath) + deleted_count += 1 + + return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count} + except Exception as e: + logger.error(f"Error deleting debug dumps for site {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/app/routers/search_sites.py b/app/routers/search_sites.py index 9ae6bff..5df10bc 100644 --- a/app/routers/search_sites.py +++ b/app/routers/search_sites.py @@ -41,17 +41,17 @@ async def update_site( ): """ Met à jour un site de recherche. - Permet uniquement de modifier is_active et priority. + Permet uniquement de modifier is_active, priority et debug_enabled. """ # Limiter les champs modifiables - allowed_fields = {"is_active", "priority"} + allowed_fields = {"is_active", "priority", "debug_enabled"} update_data = site.model_dump(exclude_unset=True) filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields} if not filtered_data: raise HTTPException( status_code=400, - detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés" + detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés" ) updated = search_service.update_site(db, site_id, filtered_data) diff --git a/app/schemas.py b/app/schemas.py index 2f82e75..e34886c 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel): is_active: bool = True priority: int = 0 requires_js: bool = False + debug_enabled: bool = False # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query} product_link_selector: str | None = None # Sélecteur CSS pour les liens produits @@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel): is_active: bool | None = None priority: int | None = None requires_js: bool | None = None + debug_enabled: bool | None = None # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None product_link_selector: str | None = None @@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel): is_active: bool priority: int requires_js: bool + debug_enabled: bool = False # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None product_link_selector: str | None = None diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py index 995c39f..f937bc8 100644 --- a/app/services/direct_search_service.py +++ b/app/services/direct_search_service.py @@ -631,6 +631,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Nettoyer le domaine (enlever protocole, www, slash final) domain = _clean_domain(raw_domain) + # Check if debug is enabled for this site + debug_enabled = site.get("debug_enabled", False) + search_url = site.get("search_url") or _get_default_search_url(domain) product_selector = site.get("product_link_selector") or _get_default_product_selector(domain) wait_selector = _get_default_wait_selector(domain) @@ -733,7 +736,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Check for Amazon blocking (503, "Toutes nos excuses", etc.) if is_amazon and _is_amazon_blocked(html_content): logger.warning(f"Amazon blocked request (attempt {attempt + 1}/{max_retries}) for '{query}'") - _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}") + # Always dump for Amazon blocking (critical error) + if debug_enabled or is_amazon: + _dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}") # Close resources before retry try: if page: @@ -747,7 +752,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Check for CAPTCHA if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content: logger.warning(f"CAPTCHA detected on {domain}!") - _dump_debug_html(html_content, domain, query) + # Always dump for CAPTCHA (critical error) + if debug_enabled or is_amazon: + _dump_debug_html(html_content, domain, f"{query}_captcha") if is_amazon and attempt < max_retries - 1: try: if page: @@ -838,8 +845,11 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 return results # No results found - for Amazon this might be temporary blocking - logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.") - _dump_debug_html(html_content, domain, query) + if debug_enabled: + logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.") + _dump_debug_html(html_content, domain, f"{query}_no_results") + else: + logger.warning(f"{domain}: 0 results found (debug disabled, no HTML dump)") # For Amazon, if no results and we have retries left, try again if is_amazon and attempt < max_retries - 1: diff --git a/app/services/search_service.py b/app/services/search_service.py index 4b6f66b..249ba72 100644 --- a/app/services/search_service.py +++ b/app/services/search_service.py @@ -76,6 +76,7 @@ async def search_products( "search_url": site.search_url, "product_link_selector": site.product_link_selector, "requires_js": site.requires_js, + "debug_enabled": site.debug_enabled, } for site in sites ] diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx index abb631b..d08fcba 100644 --- a/frontend/src/pages/Admin.jsx +++ b/frontend/src/pages/Admin.jsx @@ -437,13 +437,51 @@ export default function Admin() { // Site functions const toggleSiteActive = async (site) => { try { - await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }); + await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() }); fetchAll(); } catch (error) { toast.error('Erreur lors de la mise à jour'); } }; + const toggleSiteDebug = async (site) => { + try { + await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() }); + toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé'); + fetchAll(); + } catch (error) { + toast.error('Erreur lors de la mise à jour du debug'); + } + }; + + const fetchSiteDebugFiles = async (domain) => { + try { + const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() }); + return response.data; + } catch (error) { + console.error('Error fetching debug files:', error); + return []; + } + }; + + const downloadDebugFile = async (filename) => { + try { + window.open(`${API_URL}/debug/dumps/${filename}`, '_blank'); + } catch (error) { + toast.error('Erreur lors du téléchargement'); + } + }; + + const deleteSiteDebugFiles = async (domain) => { + if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return; + try { + const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() }); + toast.success(response.data.message); + } catch (error) { + toast.error('Erreur lors de la suppression'); + } + }; + const resetSitesToDefaults = async () => { if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return; setResettingSites(true); @@ -862,11 +900,35 @@ export default function Admin() { toggleSiteActive(site)} className="ml-2" /> - {site.category && ( - - {site.category} - - )} +
+ {site.category && ( + + {site.category} + + )} +
+ + {site.debug_enabled && ( + + )} +
+
))}