mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-21 22:28:11 +02:00
Merge pull request #163 from R0m1k3/claude/amazon-france-search-page-01KmPqbdPCqxWXxEoFw9jgFo
feat: Load Amazon homepage first to establish session
This commit is contained in:
1 file changed
+21
-3
@@ -127,9 +127,27 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
|
|||||||
products = []
|
products = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Use browserless service to get page content
|
# STRATEGY: Load Amazon homepage FIRST to establish session/cookies
|
||||||
# Try WITHOUT proxy first - some proxies may be banned by Amazon
|
# Then do the search - appears more human-like
|
||||||
logger.info("🚀 Fetching page with Browserless (NO PROXY for testing)...")
|
logger.info("🏠 Loading Amazon homepage first to establish session...")
|
||||||
|
home_html, _ = await browserless_service.get_page_content(
|
||||||
|
url="https://www.amazon.fr",
|
||||||
|
use_proxy=False,
|
||||||
|
wait_selector=None,
|
||||||
|
extract_text=False
|
||||||
|
)
|
||||||
|
|
||||||
|
if not home_html or len(home_html) < 10000:
|
||||||
|
logger.warning(f"⚠️ Homepage load failed ({len(home_html) if home_html else 0} bytes)")
|
||||||
|
else:
|
||||||
|
logger.info(f"✅ Homepage loaded ({len(home_html)} bytes) - cookies established")
|
||||||
|
|
||||||
|
# Small delay to appear more human
|
||||||
|
import asyncio
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
|
# NOW do the search
|
||||||
|
logger.info("🚀 Fetching search page with Browserless (NO PROXY)...")
|
||||||
html_content, _ = await browserless_service.get_page_content(
|
html_content, _ = await browserless_service.get_page_content(
|
||||||
url=search_url,
|
url=search_url,
|
||||||
use_proxy=False, # Try without proxy first
|
use_proxy=False, # Try without proxy first
|
||||||
|
|||||||
Reference in new issue
Block a user