test: Try Amazon scraping without proxy

- Disabled proxy (use_proxy=False) for testing
- Previous attempt got 503 from Amazon with proxy
- Will test if direct connection works better
- Can re-enable proxy later if needed
This commit is contained in:
Claude committed 2025-11-30 10:00:24 +00:00
1 parent af7c32a4d8
commit 77bc8ececf
1 file changed
+3 -3
+3 -3
View File
@@ -128,11 +128,11 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
try:
# Use browserless service to get page content
# use_proxy=True for Amazon to avoid rate limiting
logger.info("🚀 Fetching page with Browserless...")
# Try WITHOUT proxy first - some proxies may be banned by Amazon
logger.info("🚀 Fetching page with Browserless (NO PROXY for testing)...")
html_content, _ = await browserless_service.get_page_content(
url=search_url,
use_proxy=True, # Use proxy rotation for Amazon
use_proxy=False, # Try without proxy first
wait_selector=None, # Let it load naturally
extract_text=False # We want HTML for parsing
)