mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-12 01:39:25 +02:00
Merge pull request #161 from R0m1k3/claude/amazon-france-search-page-01KmPqbdPCqxWXxEoFw9jgFo
test: Try Amazon scraping without proxy
This commit is contained in:
1 file changed
+3
-3
@@ -128,11 +128,11 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
|
||||
|
||||
try:
|
||||
# Use browserless service to get page content
|
||||
# use_proxy=True for Amazon to avoid rate limiting
|
||||
logger.info("🚀 Fetching page with Browserless...")
|
||||
# Try WITHOUT proxy first - some proxies may be banned by Amazon
|
||||
logger.info("🚀 Fetching page with Browserless (NO PROXY for testing)...")
|
||||
html_content, _ = await browserless_service.get_page_content(
|
||||
url=search_url,
|
||||
use_proxy=True, # Use proxy rotation for Amazon
|
||||
use_proxy=False, # Try without proxy first
|
||||
wait_selector=None, # Let it load naturally
|
||||
extract_text=False # We want HTML for parsing
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user