mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-21 06:08:12 +02:00
feat: Create persistent Amazon scraper service (ScraperService pattern)
- New amazon_scraper_service.py with persistent browser connection - Shared browser instance across requests (better session management) - Auto-reconnect if browser connection drops - Better stealth mode (webdriver undefined, chrome runtime) - Proper cookie/popup handling for Amazon - Uses Playwright via Browserless (not Crawl4AI) - Should fix 503 errors with persistent session Based on user's ScraperService pattern for reliability
This commit is contained in:
2 files changed
+422
-3
No files matched your search
@@ -12,7 +12,7 @@ from fastapi import APIRouter, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.services.amazon_scraper_v2 import scrape_amazon_search, AmazonProduct
|
||||
from app.services.amazon_scraper_service import amazon_scraper_service, AmazonProduct
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -55,9 +55,9 @@ async def search_amazon(
|
||||
)
|
||||
yield f"event: progress\ndata: {progress.model_dump_json()}\n\n"
|
||||
|
||||
# Execute search
|
||||
# Execute search with persistent browser service
|
||||
logger.info(f"Starting Amazon search for: {q}")
|
||||
products = await scrape_amazon_search(q, max_results=max_results)
|
||||
products = await amazon_scraper_service.scrape_search(q, max_results=max_results)
|
||||
|
||||
# Convert to dict
|
||||
results = [p.model_dump() for p in products]
|
||||
|
||||
Reference in new issue
Block a user