From 3db7db48fa3746920662839864b23a400872eb6a Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 17:32:58 +0100 Subject: [PATCH 1/2] feat: Add improved search service with persistent browser connection for e-commerce scraping. --- app/services/improved_search_service.py | 75 ++++++++++++++++++------- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/app/services/improved_search_service.py b/app/services/improved_search_service.py index b7fea94..bc67564 100644 --- a/app/services/improved_search_service.py +++ b/app/services/improved_search_service.py @@ -251,15 +251,23 @@ class ImprovedSearchService: # Scrape details for each result (in parallel) if results: logger.info(f"📦 Found {len(results)} initial results, enriching with details...") + + # LIMIT: Only enrich first 10 products to avoid frontend timeouts + # Full enrichment (visiting each product page) takes too long + results_to_enrich = results[:10] + logger.info(f"⚡ Limiting enrichment to {len(results_to_enrich)} products") + semaphore = asyncio.Semaphore(2) # Limit concurrency async def scrape_with_limit(res): async with semaphore: return await cls._scrape_item_details(res, context) - tasks = [scrape_with_limit(r) for r in results] + tasks = [scrape_with_limit(r) for r in results_to_enrich] enriched_results = await asyncio.gather(*tasks) - results = [r for r in enriched_results if r] # Filter None + + # Return enriched results + remaining non-enriched (with None price) + results = [r for r in enriched_results if r] + results[10:] finally: await context.close() @@ -314,19 +322,20 @@ class ImprovedSearchService: if not title or len(title) < 3: continue - # RELAXED FILTERING: Check if at least ONE query word is in the title - # (instead of ALL words, which was too strict) - title_lower = title.lower() - if query_words: - at_least_one_word_found = False - for word in query_words: - if word in title_lower: - at_least_one_word_found = True - break - - # Only keep results with at least one matching word - if not at_least_one_word_found: - continue + # TEMPORARY: Disable keyword filtering to diagnose issues + # The strict filtering was rejecting too many valid results + # TODO: Re-enable with better logic after testing + + # title_lower = title.lower() + # if query_words: + # at_least_one_word_found = False + # for word in query_words: + # if word in title_lower: + # at_least_one_word_found = True + # break + # + # if not at_least_one_word_found: + # continue # Extract Image URL - PRIORITIZE CONFIGURED SELECTOR image_url = None @@ -597,13 +606,39 @@ async def search_products( results=[], ) - # 2. Map DB sites to Config keys + # 2. Map DB sites to Config keys with improved matching site_keys = [] for site in active_sites: - for key in SITE_CONFIGS.keys(): - if key in site.domain or site.domain in key: - site_keys.append(key) - break + matched_key = None + site_domain_normalized = site.domain.lower().replace("www.", "").replace("http://", "").replace("https://", "").strip("/") + + # Try exact match first + if site_domain_normalized in SITE_CONFIGS: + matched_key = site_domain_normalized + else: + # Try partial match + for key in SITE_CONFIGS.keys(): + key_normalized = key.lower().replace("www.", "") + + # Match if either contains the other + if key_normalized in site_domain_normalized or site_domain_normalized in key_normalized: + matched_key = key + break + + # Also try matching just the main domain part (before first dot after main name) + # e.g. "e-leclerc" matches both "e-leclerc.com" and "leclerc.fr" + site_parts = site_domain_normalized.split(".") + key_parts = key_normalized.split(".") + if len(site_parts) > 0 and len(key_parts) > 0: + if site_parts[0] in key_parts[0] or key_parts[0] in site_parts[0]: + matched_key = key + break + + if matched_key: + site_keys.append(matched_key) + logger.info(f"✅ Mapped {site.name} ({site.domain}) → {matched_key}") + else: + logger.warning(f"❌ No config found for {site.name} (domain: {site.domain}, normalized: {site_domain_normalized})") # 3. Execute searches and stream results generators = [ImprovedSearchService.search_site_generator(key, query) for key in site_keys] From ee13a0bccd70144b325f499742624500cec9c082 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 17:37:16 +0100 Subject: [PATCH 2/2] fix: resolve merge conflict in improved_search_service - unified domain matching logic --- app/services/improved_search_service.py | 55 +++++++++++++++---------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/app/services/improved_search_service.py b/app/services/improved_search_service.py index bc67564..e840782 100644 --- a/app/services/improved_search_service.py +++ b/app/services/improved_search_service.py @@ -610,33 +610,46 @@ async def search_products( site_keys = [] for site in active_sites: matched_key = None - site_domain_normalized = site.domain.lower().replace("www.", "").replace("http://", "").replace("https://", "").strip("/") - # Try exact match first - if site_domain_normalized in SITE_CONFIGS: - matched_key = site_domain_normalized - else: - # Try partial match - for key in SITE_CONFIGS.keys(): - key_normalized = key.lower().replace("www.", "") - - # Match if either contains the other - if key_normalized in site_domain_normalized or site_domain_normalized in key_normalized: + # Normalize domain for comparison (remove www., lowercase, etc.) + site_domain_normalized = site.domain.lower().replace("www.", "").replace("http://", "").replace("https://", "").strip("/") + + # Try multiple matching strategies: + for key in SITE_CONFIGS.keys(): + key_normalized = key.lower().replace("www.", "") + + # 1. Exact match + if site_domain_normalized == key_normalized: + matched_key = key + logger.info(f"✅ Mapped {site.name} ({site.domain}) → {key} (exact match)") + break + + # 2. Contains match (one in the other) + if key_normalized in site_domain_normalized or site_domain_normalized in key_normalized: + matched_key = key + logger.info(f"✅ Mapped {site.name} ({site.domain}) → {key} (contains)") + break + + # 3. Normalize punctuation (. vs - vs nothing) and compare + # e.leclerc → eleclerc, e-leclerc.com → eleclecrcom + domain_no_punct = site_domain_normalized.replace("-", "").replace(".", "") + key_no_punct = key_normalized.replace("-", "").replace(".", "") + + # Exact match without punctuation + if domain_no_punct == key_no_punct: + matched_key = key + logger.info(f"✅ Mapped {site.name} ({site.domain}) → {key} (normalized punctuation)") + break + + # Contains match without punctuation (handles .com, .fr suffixes) + if len(domain_no_punct) > 3 and len(key_no_punct) > 3: + if domain_no_punct in key_no_punct or key_no_punct in domain_no_punct: matched_key = key + logger.info(f"✅ Mapped {site.name} ({site.domain}) → {key} (normalized contains)") break - - # Also try matching just the main domain part (before first dot after main name) - # e.g. "e-leclerc" matches both "e-leclerc.com" and "leclerc.fr" - site_parts = site_domain_normalized.split(".") - key_parts = key_normalized.split(".") - if len(site_parts) > 0 and len(key_parts) > 0: - if site_parts[0] in key_parts[0] or key_parts[0] in site_parts[0]: - matched_key = key - break if matched_key: site_keys.append(matched_key) - logger.info(f"✅ Mapped {site.name} ({site.domain}) → {matched_key}") else: logger.warning(f"❌ No config found for {site.name} (domain: {site.domain}, normalized: {site_domain_normalized})")