mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-24 15:48:11 +02:00
feat: Implement improved search service with centralized configuration and site-specific parsers for various retailers.
This commit is contained in:
1 parent
a8bffaec08
commit
c2f042beee
17 files changed
+829
-476
No files matched your search
@@ -258,12 +258,11 @@ class ImprovedSearchService:
|
||||
if results:
|
||||
logger.info(f"📦 Found {len(results)} initial results, enriching with details...")
|
||||
|
||||
# LIMIT: Only enrich first 10 products to avoid frontend timeouts
|
||||
# Full enrichment (visiting each product page) takes too long
|
||||
results_to_enrich = results[:10]
|
||||
logger.info(f"⚡ Limiting enrichment to {len(results_to_enrich)} products")
|
||||
# Enrich all results
|
||||
results_to_enrich = results
|
||||
logger.info(f"⚡ Enriching all {len(results_to_enrich)} products")
|
||||
|
||||
semaphore = asyncio.Semaphore(2) # Limit concurrency
|
||||
semaphore = asyncio.Semaphore(4) # Increased concurrency slightly
|
||||
|
||||
async def scrape_with_limit(res):
|
||||
async with semaphore:
|
||||
@@ -272,8 +271,8 @@ class ImprovedSearchService:
|
||||
tasks = [scrape_with_limit(r) for r in results_to_enrich]
|
||||
enriched_results = await asyncio.gather(*tasks)
|
||||
|
||||
# Return enriched results + remaining non-enriched (with None price)
|
||||
results = [r for r in enriched_results if r] + results[10:]
|
||||
# Filter out failed enrichments if any (though scrape_item_details returns original on failure)
|
||||
results = [r for r in enriched_results if r]
|
||||
|
||||
finally:
|
||||
await context.close()
|
||||
|
||||
Reference in new issue
Block a user