From f20bae52bdcecbda3dce8eefe3d23837cccfca02 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 16:27:02 +0100 Subject: [PATCH] feat: introduce core search configuration for sites, proxies, and user agents, alongside selector diagnosis and HTML dumping utilities. --- app/core/search_config.py | 44 ++++++++++----------- diagnose_selectors.py | 71 +++++++++++++++++++++++++++++++++ dump_html.py | 83 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 22 deletions(-) create mode 100644 diagnose_selectors.py create mode 100644 dump_html.py diff --git a/app/core/search_config.py b/app/core/search_config.py index 036e336..6fa3c26 100644 --- a/app/core/search_config.py +++ b/app/core/search_config.py @@ -74,9 +74,9 @@ SITE_CONFIGS = { "stokomani.fr": { "name": "Stokomani", "search_url": "https://www.stokomani.fr/search?options%5Bprefix%5D=last&q={query}", - "product_selector": "a.reversed-link.block", - "product_image_selector": "img.object-cover, img[loading='lazy'], picture img", - "wait_selector": "a.reversed-link.block", + "product_selector": "a.reversed-link.block, a[href*='/products/']", + "product_image_selector": "img[loading='lazy'], img[class*='object'], img[sizes]", + "wait_selector": "a.reversed-link.block, .product-card", "category": "Discount", "requires_proxy": False, }, @@ -92,9 +92,9 @@ SITE_CONFIGS = { "cdiscount.com": { "name": "Cdiscount", "search_url": "https://www.cdiscount.com/search/10/{query}.html", - "product_selector": "a.prdtBImg[href*='/f-'][href*='.html']", - "product_image_selector": "img.prdtBImg", - "wait_selector": ".prdtBILDetails, .product-list", + "product_selector": "a.prdtBILnk, a[href*='/f-'][href*='.html']", + "product_image_selector": "img.lazy, img[data-src], img[src*='image']", + "wait_selector": ".prdtBILDetails, .prdtBIL", "category": "E-commerce", "requires_proxy": False, }, @@ -139,18 +139,18 @@ SITE_CONFIGS = { "centrakor.com": { "name": "Centrakor", "search_url": "https://www.centrakor.com/search/{query}", - "product_selector": "a.link.link--block", - "product_image_selector": "img.product-card__image", - "wait_selector": ".product-list, .product-item", + "product_selector": "a.link.link--block, a.product-item__link", + "product_image_selector": "img.product-card__image, img[loading='lazy'], noscript img", + "wait_selector": ".product-list, .collection__products", "category": "Discount", "requires_proxy": False, }, "lincroyable.fr": { "name": "L'Incroyable", - "search_url": "https://www.lincroyable.fr/recherche-query={query}/", - "product_selector": "a[href^='/p']", - "product_image_selector": "img.product-image", - "wait_selector": ".product-item, .product-list", + "search_url": "https://www.lincroyable.fr/recherche?query={query}", + "product_selector": "a.product-link, a[href*='/p/']", + "product_image_selector": "img[loading='lazy'], img.product-image__img, img[src*='product']", + "wait_selector": ".product-card, .products-grid", "category": "Discount", "requires_proxy": False, }, @@ -158,27 +158,27 @@ SITE_CONFIGS = { "e-leclerc.com": { "name": "E.Leclerc", "search_url": "https://www.e-leclerc.com/recherche?text={query}", - "product_selector": "a[href*='/produit'], a.product-card, article a", - "product_image_selector": "img[loading='lazy'], img[src*='scene7'], picture img", - "wait_selector": ".product-card, .search-results, article", + "product_selector": "a[data-testid='product-link'], a[href*='/cat/'][href*='/sku/']", + "product_image_selector": "img[data-testid='product-image'], img[src*='scene7'], img[loading='eager']", + "wait_selector": "[data-testid='product-card'], [data-testid='product-link']", "category": "Grande Surface", "requires_proxy": False, }, "auchan.fr": { "name": "Auchan", "search_url": "https://www.auchan.fr/search?text={query}", - "product_selector": "a[href*='/p-'], a.product-card, a[data-testid*='product']", - "product_image_selector": "img.product-image, img[loading='lazy'], picture img", - "wait_selector": ".product-card, .product-item, [data-testid*='product']", + "product_selector": "article.product-item a.product-item__name, a[class*='product-item'][href]", + "product_image_selector": "img.product-thumbnail__img, img[class*='product'][class*='img']", + "wait_selector": "article.product-item, .product-list", "category": "Grande Surface", "requires_proxy": False, }, "carrefour.fr": { "name": "Carrefour", "search_url": "https://www.carrefour.fr/s?q={query}", - "product_selector": "a[data-testid='product-card-link'], a[href*='/p/']", - "product_image_selector": "img[data-testid='product-card-image'], img[loading='lazy'], picture img", - "wait_selector": "[data-testid='product-card'], .product-card", + "product_selector": "a[data-testid='product-card-link'], a.product-card__link", + "product_image_selector": "img[data-testid='product-card-image'], img[fetchpriority], img.ds-product-image__img", + "wait_selector": "[data-testid='product-card'], article[data-testid]", "category": "Grande Surface", "requires_proxy": False, }, diff --git a/diagnose_selectors.py b/diagnose_selectors.py new file mode 100644 index 0000000..362b255 --- /dev/null +++ b/diagnose_selectors.py @@ -0,0 +1,71 @@ +""" +Quick diagnostic script to analyze actual HTML structure of search pages +""" +import asyncio +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) + +from app.services.search_service import NewSearchService +from app.core.search_config import SITE_CONFIGS + +async def diagnose_site(site_key: str, query: str = "chaise"): + """Run a search and show detailed diagnostics""" + config = SITE_CONFIGS.get(site_key) + if not config: + print(f"āŒ Site '{site_key}' not found in config") + return + + print(f"\n{'='*80}") + print(f"šŸ” Diagnosing: {config['name']} ({site_key})") + print(f"{'='*80}") + print(f"Search URL: {config['search_url'].format(query=query)}") + print(f"Product Selector: {config['product_selector']}") + print(f"Image Selector: {config.get('product_image_selector', 'NONE')}") + + try: + results = await NewSearchService.search_site(site_key, query) + + print(f"\nšŸ“Š Results: {len(results)} products found") + + if len(results) == 0: + print("āš ļø NO RESULTS - Check if product_selector is correct") + else: + print("\nāœ… Sample Results:") + for i, result in enumerate(results[:3], 1): + print(f"\n {i}. {result.title[:60]}") + print(f" URL: {result.url[:80]}") + print(f" Image: {result.image_url[:80] if result.image_url else 'āŒ NONE'}") + print(f" Price: {result.price}ā‚¬" if result.price else " Price: āŒ NONE") + + # Count images + with_images = sum(1 for r in results if r.image_url) + print(f"\nšŸ“ˆ Images: {with_images}/{len(results)} ({with_images/len(results)*100:.0f}%)") + + with_prices = sum(1 for r in results if r.price) + print(f"šŸ’° Prices: {with_prices}/{len(results)} ({with_prices/len(results)*100:.0f}%)") + + except Exception as e: + print(f"āŒ ERROR: {e}") + import traceback + traceback.print_exc() + +async def main(): + """Diagnose problematic sites""" + sites_to_check = [ + "e-leclerc.com", + "auchan.fr", + "carrefour.fr", + "stokomani.fr", + "centrakor.com", + "cdiscount.com", + "lincroyable.fr" + ] + + for site_key in sites_to_check: + await diagnose_site(site_key) + print("\n" + "="*80 + "\n") + await asyncio.sleep(1) # Rate limiting + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/dump_html.py b/dump_html.py new file mode 100644 index 0000000..f0d0d72 --- /dev/null +++ b/dump_html.py @@ -0,0 +1,83 @@ +""" +Quick HTML Dumper - Saves raw HTML from search pages for manual analysis +""" +import asyncio +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) + +from app.services.improved_search_service import ImprovedSearchService +from app.core.search_config import SITE_CONFIGS + +async def dump_search_html(site_key: str, query: str = "chaise"): + """Download and save raw HTML for manual inspection""" + config = SITE_CONFIGS.get(site_key) + if not config: + print(f"āŒ Site '{site_key}' not found") + return + + print(f"\nšŸ” Dumping HTML for: {config['name']}") + + # Ensure browser is initialized + await ImprovedSearchService.initialize() + + # Create context manually to get HTML + context = await ImprovedSearchService._create_context(ImprovedSearchService._browser) + page = await context.new_page() + + try: + search_url = config["search_url"].format(query=query) + print(f" URL: {search_url}") + + await page.goto(search_url, wait_until="networkidle", timeout=30000) + await ImprovedSearchService._handle_popups(page) + await page.wait_for_timeout(3000) + + html = await page.content() + + filename = f"dump_{site_key.replace('.', '_')}.html" + with open(filename, "w", encoding="utf-8") as f: + f.write(html) + + print(f" āœ… Saved to: {filename} ({len(html)} bytes)") + + # Quick analysis + from bs4 import BeautifulSoup + soup = BeautifulSoup(html, "html.parser") + + # Try current selector + current_selector = config.get("product_selector") + matches = soup.select(current_selector) + print(f" šŸ“Š Current selector '{current_selector}' matches: {len(matches)}") + + # Try image selector + if "product_image_selector" in config: + img_selector = config["product_image_selector"] + img_matches = soup.select(img_selector) + print(f" šŸ–¼ļø Current image selector '{img_selector}' matches: {len(img_matches)}") + + finally: + await context.close() + +async def main(): + sites = [ + "e-leclerc.com", + "auchan.fr", + "carrefour.fr", + "stokomani.fr", + "centrakor.com", + "cdiscount.com", + "lincroyable.fr" + ] + + for site_key in sites: + try: + await dump_search_html(site_key) + except Exception as e: + print(f"āŒ Error: {e}") + await asyncio.sleep(1) + + await ImprovedSearchService.shutdown() + +if __name__ == "__main__": + asyncio.run(main())