Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-11 17:29:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add per-site debug toggle for HTML dump control

Browse Source 
- Add debug_enabled field to SearchSite model
- Update schemas and API routes to allow debug toggle
- Add debug toggle button on site cards in Admin UI
- Only dump HTML when debug is enabled for the site (except for critical errors like Amazon blocking)
- Add API endpoints to list/delete debug files per site domain
- Add button to clear debug files for individual sites
This commit is contained in:
Claude committed 2025-11-24 08:01:39 +00:00
1 parent 7243bcfaa1
commit e62117f9b9
7 files changed
+157 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/models.py
+1
View File
@@ -94,6 +94,7 @@ class SearchSite(Base):
is_active: bool = Column(Boolean, default=True) # type: ignore
priority: int = Column(Integer, default=0) # type: ignore # Ordre d'affichage
requires_js: bool = Column(Boolean, default=False) # type: ignore # Force Browserless si True
debug_enabled: bool = Column(Boolean, default=False) # type: ignore # Activer le dump HTML pour ce site
price_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour le prix
search_url: str | None = Column(String, nullable=True) # type: ignore # URL de recherche avec {query} placeholder
product_link_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour les liens produits
app/routers/debug.py
+67 -4
View File
@@ -75,19 +75,82 @@ def delete_debug_dump(filename: str):
"""Delete a specific debug dump file"""
if not os.path.exists(DEBUG_DIR):
raise HTTPException(status_code=404, detail="Debug directory not found")
filepath = os.path.join(DEBUG_DIR, filename)
# Security check: prevent directory traversal
if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)):
raise HTTPException(status_code=403, detail="Access denied")
if not os.path.isfile(filepath):
raise HTTPException(status_code=404, detail="File not found")
try:
os.remove(filepath)
return {"message": "File deleted"}
except Exception as e:
logger.error(f"Error deleting debug dump {filename}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/dumps/site/{domain}", response_model=List[DebugFile])
def list_debug_dumps_by_site(domain: str):
"""List debug dump files for a specific site domain"""
if not os.path.exists(DEBUG_DIR):
return []
# Clean domain for comparison (remove www. if present)
clean_domain = domain.lower().strip()
if clean_domain.startswith("www."):
clean_domain = clean_domain[4:]
files = []
try:
for filename in os.listdir(DEBUG_DIR):
# Check if filename starts with the domain
filename_lower = filename.lower()
# Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html
if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
filepath = os.path.join(DEBUG_DIR, filename)
if os.path.isfile(filepath):
stat = os.stat(filepath)
files.append(DebugFile(
filename=filename,
size=stat.st_size,
created_at=stat.st_ctime,
created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S")
))
# Sort by creation time (newest first)
files.sort(key=lambda x: x.created_at, reverse=True)
return files
except Exception as e:
logger.error(f"Error listing debug dumps for site {domain}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/dumps/site/{domain}")
def delete_debug_dumps_by_site(domain: str):
"""Delete all debug dump files for a specific site domain"""
if not os.path.exists(DEBUG_DIR):
return {"message": "No files found", "deleted": 0}
# Clean domain for comparison
clean_domain = domain.lower().strip()
if clean_domain.startswith("www."):
clean_domain = clean_domain[4:]
deleted_count = 0
try:
for filename in os.listdir(DEBUG_DIR):
filename_lower = filename.lower()
if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")):
filepath = os.path.join(DEBUG_DIR, filename)
if os.path.isfile(filepath):
os.remove(filepath)
deleted_count += 1
return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count}
except Exception as e:
logger.error(f"Error deleting debug dumps for site {domain}: {e}")
raise HTTPException(status_code=500, detail=str(e))
app/routers/search_sites.py
+3 -3
View File
@@ -41,17 +41,17 @@ async def update_site(
):
"""
Met à jour un site de recherche.
Permet uniquement de modifier is_active et priority.
Permet uniquement de modifier is_active, priority et debug_enabled.
"""
# Limiter les champs modifiables
allowed_fields = {"is_active", "priority"}
allowed_fields = {"is_active", "priority", "debug_enabled"}
update_data = site.model_dump(exclude_unset=True)
filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields}
if not filtered_data:
raise HTTPException(
status_code=400,
detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés"
detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés"
)
updated = search_service.update_site(db, site_id, filtered_data)
app/schemas.py
+3
View File
@@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel):
is_active: bool = True
priority: int = 0
requires_js: bool = False
debug_enabled: bool = False # Activer le dump HTML pour ce site
price_selector: str | None = None
search_url: str | None = None # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query}
product_link_selector: str | None = None # Sélecteur CSS pour les liens produits
@@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel):
is_active: bool | None = None
priority: int | None = None
requires_js: bool | None = None
debug_enabled: bool | None = None # Activer le dump HTML pour ce site
price_selector: str | None = None
search_url: str | None = None
product_link_selector: str | None = None
@@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel):
is_active: bool
priority: int
requires_js: bool
debug_enabled: bool = False # Activer le dump HTML pour ce site
price_selector: str | None = None
search_url: str | None = None
product_link_selector: str | None = None
app/services/direct_search_service.py
+14 -4
View File
@@ -631,6 +631,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
# Nettoyer le domaine (enlever protocole, www, slash final)
domain = _clean_domain(raw_domain)
# Check if debug is enabled for this site
debug_enabled = site.get("debug_enabled", False)
search_url = site.get("search_url") or _get_default_search_url(domain)
product_selector = site.get("product_link_selector") or _get_default_product_selector(domain)
wait_selector = _get_default_wait_selector(domain)
@@ -733,7 +736,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
# Check for Amazon blocking (503, "Toutes nos excuses", etc.)
if is_amazon and _is_amazon_blocked(html_content):
logger.warning(f"Amazon blocked request (attempt {attempt + 1}/{max_retries}) for '{query}'")
_dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
# Always dump for Amazon blocking (critical error)
if debug_enabled or is_amazon:
_dump_debug_html(html_content, domain, f"{query}_blocked_attempt{attempt + 1}")
# Close resources before retry
try:
if page:
@@ -747,7 +752,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
# Check for CAPTCHA
if "Enter the characters you see below" in html_content or "Saisissez les caractères" in html_content:
logger.warning(f"CAPTCHA detected on {domain}!")
_dump_debug_html(html_content, domain, query)
# Always dump for CAPTCHA (critical error)
if debug_enabled or is_amazon:
_dump_debug_html(html_content, domain, f"{query}_captcha")
if is_amazon and attempt < max_retries - 1:
try:
if page:
@@ -838,8 +845,11 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
return results
# No results found - for Amazon this might be temporary blocking
logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
_dump_debug_html(html_content, domain, query)
if debug_enabled:
logger.warning(f"{domain}: 0 results found. Dumping HTML for debugging.")
_dump_debug_html(html_content, domain, f"{query}_no_results")
else:
logger.warning(f"{domain}: 0 results found (debug disabled, no HTML dump)")
# For Amazon, if no results and we have retries left, try again
if is_amazon and attempt < max_retries - 1:
app/services/search_service.py
+1
View File
@@ -76,6 +76,7 @@ async def search_products(
"search_url": site.search_url,
"product_link_selector": site.product_link_selector,
"requires_js": site.requires_js,
"debug_enabled": site.debug_enabled,
}
for site in sites
]
frontend/src/pages/Admin.jsx
+68 -6
View File
@@ -437,13 +437,51 @@ export default function Admin() {
// Site functions
const toggleSiteActive = async (site) => {
try {
await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active });
await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() });
fetchAll();
} catch (error) {
toast.error('Erreur lors de la mise à jour');
}
};
const toggleSiteDebug = async (site) => {
try {
await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() });
toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé');
fetchAll();
} catch (error) {
toast.error('Erreur lors de la mise à jour du debug');
}
};
const fetchSiteDebugFiles = async (domain) => {
try {
const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
return response.data;
} catch (error) {
console.error('Error fetching debug files:', error);
return [];
}
};
const downloadDebugFile = async (filename) => {
try {
window.open(`${API_URL}/debug/dumps/${filename}`, '_blank');
} catch (error) {
toast.error('Erreur lors du téléchargement');
}
};
const deleteSiteDebugFiles = async (domain) => {
if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return;
try {
const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() });
toast.success(response.data.message);
} catch (error) {
toast.error('Erreur lors de la suppression');
}
};
const resetSitesToDefaults = async () => {
if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return;
setResettingSites(true);
@@ -862,11 +900,35 @@ export default function Admin() {
</div>
<Switch checked={site.is_active} onCheckedChange={() => toggleSiteActive(site)} className="ml-2" />
</div>
{site.category && (
<span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
{site.category}
</span>
)}
<div className="flex items-center justify-between">
{site.category && (
<span className="inline-flex items-center rounded-full border px-2 py-0.5 text-xs bg-secondary">
{site.category}
</span>
)}
<div className="flex items-center gap-2 ml-auto">
<Button
variant="ghost"
size="sm"
className={cn("h-7 px-2", site.debug_enabled && "text-orange-500")}
onClick={() => toggleSiteDebug(site)}
title={site.debug_enabled ? "Désactiver le debug" : "Activer le debug"}
>
<Bug className="h-4 w-4" />
</Button>
{site.debug_enabled && (
<Button
variant="ghost"
size="sm"
className="h-7 px-2"
onClick={() => deleteSiteDebugFiles(site.domain)}
title="Supprimer fichiers debug"
>
<Trash2 className="h-4 w-4" />
</Button>
)}
</div>
</div>
</div>
))}
</div>