From eef4f60e49620714bdc88a318c2096c79017b8a1 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Sun, 30 Nov 2025 16:13:59 +0000
Subject: [PATCH] fix: Improve site mapping and relax GenericParser filtering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes two critical issues preventing some sites from working:

**1. Site Mapping Bug (E.Leclerc, La Foir'Fouille not searching)**
- Previous mapping logic was too strict: `if key in domain or domain in key`
- E.Leclerc in DB as "leclerc.com" didn't match config key "e-leclerc.com"
- La Foir'Fouille similar issues

Solution: Multi-strategy matching
- Normalize domains (remove www., lowercase)
- Try exact match first
- Try substring match
- Try without prefixes (e-, la-)
- Log warnings for unmatched sites

**2. GenericParser Over-Filtering (Stokomani: 48 links → 0 products)**
- Previous filter required at least one query word in title
- Too strict: filtered valid products with different wording

Solution: Very relaxed filtering
- Keep products with matching query words
- ALSO keep products with substantial titles (>15 chars) even without match
- Only filter very short titles (<15 chars) without query match
- Added debug logging to track filtering decisions

**Additional Improvements:**
- Added detailed logging for title extraction
- Better visibility of why products are filtered/kept
- Helps diagnose future parser issues

These changes should significantly improve product discovery
across all sites using GenericParser.
---
 app/services/improved_search_service.py | 35 +++++++++++++++++++++++--
 app/services/parsers/generic_parser.py  | 25 +++++++++++++++---
 2 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/app/services/improved_search_service.py b/app/services/improved_search_service.py
index 788f838..a22c155 100644
--- a/app/services/improved_search_service.py
+++ b/app/services/improved_search_service.py
@@ -620,14 +620,45 @@ async def search_products(
         results=[],
     )
 
-    # 2. Map DB sites to Config keys
+    # 2. Map DB sites to Config keys with improved matching
     site_keys = []
     for site in active_sites:
+        matched = False
+
+        # Normalize domain for comparison (remove www., e-, etc.)
+        normalized_domain = site.domain.lower().replace("www.", "").strip()
+
         for key in SITE_CONFIGS.keys():
-            if key in site.domain or site.domain in key:
+            normalized_key = key.lower().replace("www.", "").strip()
+
+            # Try multiple matching strategies:
+            # 1. Exact match
+            if normalized_domain == normalized_key:
                 site_keys.append(key)
+                matched = True
+                logger.debug(f"✓ Matched {site.domain} → {key} (exact)")
                 break
 
+            # 2. Key contains domain or domain contains key
+            if normalized_key in normalized_domain or normalized_domain in normalized_key:
+                site_keys.append(key)
+                matched = True
+                logger.debug(f"✓ Matched {site.domain} → {key} (contains)")
+                break
+
+            # 3. Remove prefixes like "e-" and try again
+            domain_without_prefix = normalized_domain.replace("e-", "").replace("la-", "")
+            key_without_prefix = normalized_key.replace("e-", "").replace("la-", "")
+
+            if domain_without_prefix == key_without_prefix:
+                site_keys.append(key)
+                matched = True
+                logger.debug(f"✓ Matched {site.domain} → {key} (without prefix)")
+                break
+
+        if not matched:
+            logger.warning(f"⚠️ No config found for site: {site.domain} (id={site.id})")
+
     # 3. Execute searches and stream results
     generators = [ImprovedSearchService.search_site_generator(key, query) for key in site_keys]
 
diff --git a/app/services/parsers/generic_parser.py b/app/services/parsers/generic_parser.py
index 866e205..65c1018 100644
--- a/app/services/parsers/generic_parser.py
+++ b/app/services/parsers/generic_parser.py
@@ -94,11 +94,30 @@ class GenericParser(BaseParser):
                         title = img.get("alt", "")
 
                 if not title or len(title) < 3:
+                    self.logger.debug(f"Skipped: empty or too short title")
                     continue
 
-                # Filter by query (relaxed: at least one word match)
-                if not self.filter_by_query(title, query, strict=False):
-                    continue
+                self.logger.debug(f"Extracted title: '{title[:60]}'...")
+
+                # RELAXED FILTERING: Only filter out if title is completely unrelated
+                # Accept product if:
+                # 1. Query is empty/short
+                # 2. At least one query word is in title (case-insensitive)
+                # 3. Title length is reasonable (not just numbers/symbols)
+                if query and len(query) > 2:
+                    # Very permissive: keep if ANY query word matches OR title is substantial
+                    query_words = [w.lower() for w in query.split() if len(w) > 2]
+                    title_lower = title.lower()
+
+                    if query_words:
+                        # Check if at least one word matches
+                        has_match = any(word in title_lower for word in query_words)
+
+                        # If no match and title is very short, skip
+                        # But if title is substantial (>15 chars), keep it anyway (might be relevant)
+                        if not has_match and len(title) < 15:
+                            self.logger.debug(f"Filtered: '{title[:40]}' - no query match and too short")
+                            continue
 
                 # Extract image using configured selector
                 image_url = None