feat: introduce core search configuration for sites, proxies, and user agents, alongside selector diagnosis and HTML dumping utilities.

app/core/search_config.py

@@ -74,9 +74,9 @@ SITE_CONFIGS = {
     "stokomani.fr": {
         "name": "Stokomani",
         "search_url": "https://www.stokomani.fr/search?options%5Bprefix%5D=last&q={query}",
-        "product_selector": "a.reversed-link.block",
-        "product_image_selector": "img.object-cover, img[loading='lazy'], picture img",
-        "wait_selector": "a.reversed-link.block",
+        "product_selector": "a.reversed-link.block, a[href*='/products/']",
+        "product_image_selector": "img[loading='lazy'], img[class*='object'], img[sizes]",
+        "wait_selector": "a.reversed-link.block, .product-card",
         "category": "Discount",
         "requires_proxy": False,
     },
@@ -92,9 +92,9 @@ SITE_CONFIGS = {
     "cdiscount.com": {
         "name": "Cdiscount",
         "search_url": "https://www.cdiscount.com/search/10/{query}.html",
-        "product_selector": "a.prdtBImg[href*='/f-'][href*='.html']",
-        "product_image_selector": "img.prdtBImg",
-        "wait_selector": ".prdtBILDetails, .product-list",
+        "product_selector": "a.prdtBILnk, a[href*='/f-'][href*='.html']",
+        "product_image_selector": "img.lazy, img[data-src], img[src*='image']",
+        "wait_selector": ".prdtBILDetails, .prdtBIL",
         "category": "E-commerce",
         "requires_proxy": False,
     },
@@ -139,18 +139,18 @@ SITE_CONFIGS = {
     "centrakor.com": {
         "name": "Centrakor",
         "search_url": "https://www.centrakor.com/search/{query}",
-        "product_selector": "a.link.link--block",
-        "product_image_selector": "img.product-card__image",
-        "wait_selector": ".product-list, .product-item",
+        "product_selector": "a.link.link--block, a.product-item__link",
+        "product_image_selector": "img.product-card__image, img[loading='lazy'], noscript img",
+        "wait_selector": ".product-list, .collection__products",
         "category": "Discount",
         "requires_proxy": False,
     },
     "lincroyable.fr": {
         "name": "L'Incroyable",
-        "search_url": "https://www.lincroyable.fr/recherche-query={query}/",
-        "product_selector": "a[href^='/p']",
-        "product_image_selector": "img.product-image",
-        "wait_selector": ".product-item, .product-list",
+        "search_url": "https://www.lincroyable.fr/recherche?query={query}",
+        "product_selector": "a.product-link, a[href*='/p/']",
+        "product_image_selector": "img[loading='lazy'], img.product-image__img, img[src*='product']",
+        "wait_selector": ".product-card, .products-grid",
         "category": "Discount",
         "requires_proxy": False,
     },
@@ -158,27 +158,27 @@ SITE_CONFIGS = {
     "e-leclerc.com": {
         "name": "E.Leclerc",
         "search_url": "https://www.e-leclerc.com/recherche?text={query}",
-        "product_selector": "a[href*='/produit'], a.product-card, article a",
-        "product_image_selector": "img[loading='lazy'], img[src*='scene7'], picture img",
-        "wait_selector": ".product-card, .search-results, article",
+        "product_selector": "a[data-testid='product-link'], a[href*='/cat/'][href*='/sku/']",
+        "product_image_selector": "img[data-testid='product-image'], img[src*='scene7'], img[loading='eager']",
+        "wait_selector": "[data-testid='product-card'], [data-testid='product-link']",
         "category": "Grande Surface",
         "requires_proxy": False,
     },
     "auchan.fr": {
         "name": "Auchan",
         "search_url": "https://www.auchan.fr/search?text={query}",
-        "product_selector": "a[href*='/p-'], a.product-card, a[data-testid*='product']",
-        "product_image_selector": "img.product-image, img[loading='lazy'], picture img",
-        "wait_selector": ".product-card, .product-item, [data-testid*='product']",
+        "product_selector": "article.product-item a.product-item__name, a[class*='product-item'][href]",
+        "product_image_selector": "img.product-thumbnail__img, img[class*='product'][class*='img']",
+        "wait_selector": "article.product-item, .product-list",
         "category": "Grande Surface",
         "requires_proxy": False,
     },
     "carrefour.fr": {
         "name": "Carrefour",
         "search_url": "https://www.carrefour.fr/s?q={query}",
-        "product_selector": "a[data-testid='product-card-link'], a[href*='/p/']",
-        "product_image_selector": "img[data-testid='product-card-image'], img[loading='lazy'], picture img",
-        "wait_selector": "[data-testid='product-card'], .product-card",
+        "product_selector": "a[data-testid='product-card-link'], a.product-card__link",
+        "product_image_selector": "img[data-testid='product-card-image'], img[fetchpriority], img.ds-product-image__img",
+        "wait_selector": "[data-testid='product-card'], article[data-testid]",
         "category": "Grande Surface",
         "requires_proxy": False,
     },

diagnose_selectors.py

@@ -0,0 +1,71 @@
+"""
+Quick diagnostic script to analyze actual HTML structure of search pages
+"""
+import asyncio
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from app.services.search_service import NewSearchService
+from app.core.search_config import SITE_CONFIGS
+
+async def diagnose_site(site_key: str, query: str = "chaise"):
+    """Run a search and show detailed diagnostics"""
+    config = SITE_CONFIGS.get(site_key)
+    if not config:
+        print(f"❌ Site '{site_key}' not found in config")
+        return
+    
+    print(f"\n{'='*80}")
+    print(f"🔍 Diagnosing: {config['name']} ({site_key})")
+    print(f"{'='*80}")
+    print(f"Search URL: {config['search_url'].format(query=query)}")
+    print(f"Product Selector: {config['product_selector']}")
+    print(f"Image Selector: {config.get('product_image_selector', 'NONE')}")
+    
+    try:
+        results = await NewSearchService.search_site(site_key, query)
+        
+        print(f"\n📊 Results: {len(results)} products found")
+        
+        if len(results) == 0:
+            print("⚠️  NO RESULTS - Check if product_selector is correct")
+        else:
+            print("\n✅ Sample Results:")
+            for i, result in enumerate(results[:3], 1):
+                print(f"\n  {i}. {result.title[:60]}")
+                print(f"     URL: {result.url[:80]}")
+                print(f"     Image: {result.image_url[:80] if result.image_url else '❌ NONE'}")
+                print(f"     Price: {result.price}€" if result.price else "     Price: ❌ NONE")
+            
+            # Count images
+            with_images = sum(1 for r in results if r.image_url)
+            print(f"\n📈 Images: {with_images}/{len(results)} ({with_images/len(results)*100:.0f}%)")
+            
+            with_prices = sum(1 for r in results if r.price)
+            print(f"💰 Prices: {with_prices}/{len(results)} ({with_prices/len(results)*100:.0f}%)")
+            
+    except Exception as e:
+        print(f"❌ ERROR: {e}")
+        import traceback
+        traceback.print_exc()
+
+async def main():
+    """Diagnose problematic sites"""
+    sites_to_check = [
+        "e-leclerc.com",
+        "auchan.fr", 
+        "carrefour.fr",
+        "stokomani.fr",
+        "centrakor.com",
+        "cdiscount.com",
+        "lincroyable.fr"
+    ]
+    
+    for site_key in sites_to_check:
+        await diagnose_site(site_key)
+        print("\n" + "="*80 + "\n")
+        await asyncio.sleep(1)  # Rate limiting
+
+if __name__ == "__main__":
+    asyncio.run(main())

dump_html.py

@@ -0,0 +1,83 @@
+"""
+Quick HTML Dumper - Saves raw HTML from search pages for manual analysis
+"""
+import asyncio
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from app.services.improved_search_service import ImprovedSearchService
+from app.core.search_config import SITE_CONFIGS
+
+async def dump_search_html(site_key: str, query: str = "chaise"):
+    """Download and save raw HTML for manual inspection"""
+    config = SITE_CONFIGS.get(site_key)
+    if not config:
+        print(f"❌ Site '{site_key}' not found")
+        return
+    
+    print(f"\n🔍 Dumping HTML for: {config['name']}")
+    
+    # Ensure browser is initialized
+    await ImprovedSearchService.initialize()
+    
+    # Create context manually to get HTML
+    context = await ImprovedSearchService._create_context(ImprovedSearchService._browser)
+    page = await context.new_page()
+    
+    try:
+        search_url = config["search_url"].format(query=query)
+        print(f"   URL: {search_url}")
+        
+        await page.goto(search_url, wait_until="networkidle", timeout=30000)
+        await ImprovedSearchService._handle_popups(page)
+        await page.wait_for_timeout(3000)
+        
+        html = await page.content()
+        
+        filename = f"dump_{site_key.replace('.', '_')}.html"
+        with open(filename, "w", encoding="utf-8") as f:
+            f.write(html)
+        
+        print(f"   ✅ Saved to: {filename} ({len(html)} bytes)")
+        
+        # Quick analysis
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(html, "html.parser")
+        
+        # Try current selector
+        current_selector = config.get("product_selector")
+        matches = soup.select(current_selector)
+        print(f"   📊 Current selector '{current_selector}' matches: {len(matches)}")
+        
+        # Try image selector
+        if "product_image_selector" in config:
+            img_selector = config["product_image_selector"]
+            img_matches = soup.select(img_selector)
+            print(f"   🖼️  Current image selector '{img_selector}' matches: {len(img_matches)}")
+        
+    finally:
+        await context.close()
+
+async def main():
+    sites = [
+        "e-leclerc.com",
+        "auchan.fr",
+        "carrefour.fr",
+        "stokomani.fr",
+        "centrakor.com",
+        "cdiscount.com",
+        "lincroyable.fr"
+    ]
+    
+    for site_key in sites:
+        try:
+            await dump_search_html(site_key)
+        except Exception as e:
+            print(f"❌ Error: {e}")
+        await asyncio.sleep(1)
+    
+    await ImprovedSearchService.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())