Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-15 19:27:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add product search orchestration service with direct, light scraper, and browserless methods

Browse Source
This commit is contained in:
Michael committed 2025-11-28 23:57:51 +01:00
1 parent f187e6cb9d
commit b75ad636dd
1 file changed
+11 -2
Download Patch File Download Diff File Expand all files Collapse all files
app/services/search_service.py
+11 -2
View File
@@ -137,8 +137,10 @@ async def search_products(
results: list[SearchResultItem] = []
completed = 0
# Créer un sémaphore pour limiter la concurrence
# Créer un sémaphore pour limiter la concurrence globale
semaphore = asyncio.Semaphore(parallel_limit)
# Sémaphore spécifique pour Amazon (1 seul thread à la fois pour éviter le blocage)
amazon_semaphore = asyncio.Semaphore(1)
async def process_url(search_result: direct_search_service.SearchResult) -> SearchResultItem | None:
async with semaphore:
@@ -176,7 +178,14 @@ async def search_products(
# Fallback: Browserless + IA
try:
# Passer le navigateur partagé
result = await _scrape_with_browserless(url, site_name, domain, search_result.title, browser)
if "amazon" in domain.lower():
async with amazon_semaphore:
# Petit délai supplémentaire entre les requêtes Amazon
await asyncio.sleep(2.0)
result = await _scrape_with_browserless(url, site_name, domain, search_result.title, browser)
else:
result = await _scrape_with_browserless(url, site_name, domain, search_result.title, browser)
return result
except Exception as e:
logger.error(f"Erreur scraping {url}: {e}")