From 368531f0e212833a5756336177445aac6f9885e9 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Tue, 13 Jan 2026 13:14:15 +0100 Subject: [PATCH] fix: refine bot detection to avoid false positives on valid pages --- app/services/scheduler_service.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 26beda0..24a2b03 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -268,16 +268,6 @@ async def process_item_check(item_id: int): await loop.run_in_executor(None, _update_db_error, item_id, f"Amazon Bot Detection: {page_title}") return - # Generic Bot Detection for all sites - bot_terms = ["captcha", "robot or human", "bot verification", "security check", "access denied", "cloudflare"] - html_lower = html_content.lower() if html_content else "" - if any(term in page_title.lower() for term in bot_terms) or any(term in html_lower for term in bot_terms): - logger.warning(f"Bot detection / blocker triggered for item {item_id} (Title: {page_title})") - await loop.run_in_executor( - None, _update_db_error, item_id, f"Accès bloqué par le site (Bot Detection) : {page_title}" - ) - return - # Detect Action.com product unavailability is_action = "action.com" in item_data["url"] is_product_unavailable = False @@ -317,6 +307,24 @@ async def process_item_check(item_id: int): is_generic_title = _is_generic_or_error_title(page_title) if not titles_match: + # If titles don't match, check if it's because of a generic bot detection/blocker + # We only do this check if titles_match is False to avoid false positives on valid pages + bot_terms = [ + "captcha", + "robot or human", + "bot verification", + "security check", + "access denied", + "cloudflare ray id", + ] + html_lower = html_content.lower() if html_content else "" + if any(term in page_title.lower() for term in bot_terms) or any(term in html_lower for term in bot_terms): + logger.warning(f"Bot detection / blocker confirmed for item {item_id} (Title: {page_title})") + await loop.run_in_executor( + None, _update_db_error, item_id, f"Accès bloqué par le site (Bot Detection) : {page_title}" + ) + return + # If it's a generic or error title, don't mark as unavailable, just mark as error if is_generic_title: logger.warning(f"Generic/Error title detected for item {item_id}: '{page_title}'")