mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: Add Playwright-based web scraping service with advanced stealth techniques and proxy support, including Amazon-specific logic.
This commit is contained in:
Michael committed
1 parent
4ed574e814
commit
3caea8316b
3 files changed
+83
-6
No files matched your search
@@ -41,18 +41,14 @@ USER_AGENT_POOL = [
|
||||
# Chrome Windows - Latest versions
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
|
||||
# Chrome Mac
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
# Firefox Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
|
||||
# Firefox Mac
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
# Edge Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
# Linux
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
# Amazon-specific blocking indicators
|
||||
@@ -992,6 +988,18 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
|
||||
if is_amazon:
|
||||
await _random_delay(1.0, 3.0)
|
||||
|
||||
# Warmup for Amazon: Visit home page first to get cookies (only on first attempt)
|
||||
if is_amazon and attempt == 0:
|
||||
try:
|
||||
logger.info(f"Amazon Warmup: Visiting home page...")
|
||||
# Use a shorter timeout for warmup
|
||||
await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
|
||||
await asyncio.sleep(random.uniform(2.0, 4.0))
|
||||
# Simulate a bit of mouse movement
|
||||
await _simulate_human_behavior(page)
|
||||
except Exception as e:
|
||||
logger.warning(f"Amazon warmup failed (non-critical): {e}")
|
||||
|
||||
# Naviguer vers la page de recherche
|
||||
await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ USER_AGENT_POOL = [
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
# Amazon blocking indicators
|
||||
@@ -564,6 +565,14 @@ class ScraperService:
|
||||
# Add random delay before navigation for Amazon
|
||||
if is_amazon:
|
||||
await asyncio.sleep(random.uniform(1.0, 3.0))
|
||||
|
||||
# Warmup: Visit home page first
|
||||
try:
|
||||
logger.info("Amazon Warmup: Visiting home page...")
|
||||
await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
|
||||
await asyncio.sleep(random.uniform(2.0, 4.0))
|
||||
except Exception as e:
|
||||
logger.warning(f"Amazon warmup failed: {e}")
|
||||
|
||||
logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
|
||||
response = None
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
|
||||
|
||||
async def debug_amazon():
|
||||
logger.info("Starting Amazon Debug Script")
|
||||
|
||||
async with async_playwright() as p:
|
||||
try:
|
||||
logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
|
||||
browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
|
||||
|
||||
# Use a very standard, recent User-Agent
|
||||
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
|
||||
context = await browser.new_context(
|
||||
user_agent=user_agent,
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
locale="fr-FR",
|
||||
timezone_id="Europe/Paris"
|
||||
)
|
||||
|
||||
page = await context.new_page()
|
||||
|
||||
url = "https://www.amazon.fr/s?k=iphone"
|
||||
logger.info(f"Navigating to {url}")
|
||||
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
|
||||
if response:
|
||||
status = response.status
|
||||
logger.info(f"Response Status: {status}")
|
||||
|
||||
content = await page.content()
|
||||
if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content:
|
||||
logger.error("BLOCK DETECTED: Found blocking message in content")
|
||||
else:
|
||||
logger.info("No obvious blocking message found")
|
||||
|
||||
# Save screenshot
|
||||
await page.screenshot(path="debug_amazon_screenshot.png")
|
||||
logger.info("Screenshot saved to debug_amazon_screenshot.png")
|
||||
|
||||
else:
|
||||
logger.error("No response received")
|
||||
|
||||
await browser.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(debug_amazon())
|
||||
Reference in new issue
Block a user