From 5e5f49a5f69fca82ea638ec5c57770fdc6b21942 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Sun, 23 Nov 2025 09:02:51 +0000
Subject: [PATCH] fix: Improve Amazon search reliability and URL validation

Amazon selectors simplified:
- Remove overly restrictive [href*='/ref='] requirement
- Use simpler data-asin and s-result-item selectors
- Wait for [data-component-type='s-search-result']

URL validation improved (_is_non_product_url):
- For Amazon: ONLY accept URLs containing /dp/ or /gp/product/
- Reject Amazon URLs with /ref=cs_ (error page links)
- Reject Amazon redirect URLs without product paths
- Add patterns: /503, /error, /robot, /captcha
- Add French site patterns: /application, /courses

Navigation link filter improvements:
- Add more Amazon error patterns: /ref=nb_, /ref=sr_
- Add /ap/signin, /gp/yourstore, /gp/cart
---
 app/services/direct_search_service.py | 41 +++++++++++++++++++--------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py
index 568862e..b94ba5a 100644
--- a/app/services/direct_search_service.py
+++ b/app/services/direct_search_service.py
@@ -174,15 +174,14 @@ DEFAULT_SITE_CONFIGS = {
     # === E-COMMERCE GÉNÉRALISTE ===
     "amazon.fr": {
         "name": "Amazon France",
-        # URL de recherche Amazon
         "search_url": "https://www.amazon.fr/s?k={query}",
         "product_selector": (
-            # Sélecteurs très spécifiques pour Amazon (liens /dp/ avec ASIN)
-            "a[href*='/dp/'][href*='/ref=']:not([href*='logo']):not([href*='nav']), "
-            "div[data-asin]:not([data-asin='']) h2 a[href*='/dp/'], "
-            ".s-result-item[data-asin] h2 a"
+            # Sélecteurs simples pour Amazon - les produits ont data-asin
+            "div[data-asin]:not([data-asin='']) h2 a, "
+            ".s-result-item[data-asin] h2 a, "
+            "[data-component-type='s-search-result'] h2 a"
         ),
-        "wait_selector": ".s-main-slot .s-result-item[data-asin]",
+        "wait_selector": "[data-component-type='s-search-result']",
         "category": "E-commerce",
         "requires_js": True,
         "priority": 11,
@@ -229,12 +228,12 @@ DEFAULT_SITE_CONFIGS = {
         "name": "Amazon US",
         "search_url": "https://www.amazon.com/s?k={query}",
         "product_selector": (
-            # Sélecteurs très spécifiques pour Amazon (liens /dp/ avec ASIN)
-            "a[href*='/dp/'][href*='/ref=']:not([href*='logo']):not([href*='nav']), "
-            "div[data-asin]:not([data-asin='']) h2 a[href*='/dp/'], "
-            ".s-result-item[data-asin] h2 a"
+            # Sélecteurs simples pour Amazon - les produits ont data-asin
+            "div[data-asin]:not([data-asin='']) h2 a, "
+            ".s-result-item[data-asin] h2 a, "
+            "[data-component-type='s-search-result'] h2 a"
         ),
-        "wait_selector": ".s-main-slot .s-result-item[data-asin]",
+        "wait_selector": "[data-component-type='s-search-result']",
         "category": "E-commerce",
         "requires_js": True,
         "priority": 99,
@@ -751,6 +750,9 @@ def _is_navigation_link(href: str) -> bool:
         # Amazon spécifique
         "/ref=cs_", "/logo", "/nav_", "/gp/help",
         "/gp/css", "/gp/redirect", "/hz/",
+        "/ref=nb_", "/ref=sr_", "amazon.fr/ref=", "amazon.com/ref=",
+        "/ap/signin", "/gp/yourstore", "/gp/cart",
+        "/503", "/error", "/robot",
         # Liens génériques à exclure
         "/home", "/accueil", "/index",
         "/newsletter", "/subscribe",
@@ -838,13 +840,28 @@ def _clean_domain(domain: str) -> str:
 
 def _is_non_product_url(url: str) -> bool:
     """Vérifie si l'URL n'est pas une page produit"""
+    url_lower = url.lower()
+
+    # Pour Amazon, on vérifie que l'URL contient /dp/ (page produit)
+    if "amazon" in url_lower:
+        # Si c'est Amazon mais pas un lien /dp/, c'est pas un produit
+        if "/dp/" not in url_lower and "/gp/product/" not in url_lower:
+            return True
+        # Exclure les liens d'erreur/redirection Amazon
+        if "/ref=cs_" in url_lower or "/ref=nb_sb" in url_lower:
+            return True
+        if "amazon.fr/ref=" in url_lower or "amazon.com/ref=" in url_lower:
+            if "/dp/" not in url_lower:
+                return True
+
     non_product_patterns = [
         "/search", "/category", "/categories", "/brand", "/brands",
         "/help", "/contact", "/about", "/account", "/cart", "/checkout",
         "/login", "/register", "/wishlist", "/compare", "/reviews",
         "/terms", "/privacy", "/faq", "/shipping", "/returns",
+        "/503", "/error", "/robot", "/captcha",
+        "/application", "/courses", "/app/",
     ]
-    url_lower = url.lower()
     return any(pattern in url_lower for pattern in non_product_patterns)