From 29a4635995c30f2675435cc6cc6062bf9da13203 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Nov 2025 10:21:25 +0000 Subject: [PATCH] feat: Load Amazon homepage first to establish session - Load amazon.fr homepage before search to get cookies - Add 2s delay between homepage and search (human-like) - Trying to bypass 503 errors by establishing session first - Each request still creates new context (limitation) --- app/services/amazon_scraper_v2.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/app/services/amazon_scraper_v2.py b/app/services/amazon_scraper_v2.py index b65e0ed..4ba20f9 100644 --- a/app/services/amazon_scraper_v2.py +++ b/app/services/amazon_scraper_v2.py @@ -127,9 +127,27 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon products = [] try: - # Use browserless service to get page content - # Try WITHOUT proxy first - some proxies may be banned by Amazon - logger.info("🚀 Fetching page with Browserless (NO PROXY for testing)...") + # STRATEGY: Load Amazon homepage FIRST to establish session/cookies + # Then do the search - appears more human-like + logger.info("🏠 Loading Amazon homepage first to establish session...") + home_html, _ = await browserless_service.get_page_content( + url="https://www.amazon.fr", + use_proxy=False, + wait_selector=None, + extract_text=False + ) + + if not home_html or len(home_html) < 10000: + logger.warning(f"⚠️ Homepage load failed ({len(home_html) if home_html else 0} bytes)") + else: + logger.info(f"✅ Homepage loaded ({len(home_html)} bytes) - cookies established") + + # Small delay to appear more human + import asyncio + await asyncio.sleep(2) + + # NOW do the search + logger.info("🚀 Fetching search page with Browserless (NO PROXY)...") html_content, _ = await browserless_service.get_page_content( url=search_url, use_proxy=False, # Try without proxy first