Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-20 05:38:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add Playwright-based web scraping service with advanced stealth techniques and proxy support, including Amazon-specific logic.

Browse Source
This commit is contained in:
Michael committed 2025-11-28 23:53:45 +01:00
1 parent 4ed574e814
commit 3caea8316b
3 files changed
+83 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/direct_search_service.py
+14 -6
View File
@@ -41,18 +41,14 @@ USER_AGENT_POOL = [
# Chrome Windows - Latest versions
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
# Chrome Mac
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
# Firefox Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
# Firefox Mac
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) Gecko/20100101 Firefox/133.0",
# Edge Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
# Linux
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
]
# Amazon-specific blocking indicators
@@ -992,6 +988,18 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
if is_amazon:
await _random_delay(1.0, 3.0)
# Warmup for Amazon: Visit home page first to get cookies (only on first attempt)
if is_amazon and attempt == 0:
try:
logger.info(f"Amazon Warmup: Visiting home page...")
# Use a shorter timeout for warmup
await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
await asyncio.sleep(random.uniform(2.0, 4.0))
# Simulate a bit of mouse movement
await _simulate_human_behavior(page)
except Exception as e:
logger.warning(f"Amazon warmup failed (non-critical): {e}")
# Naviguer vers la page de recherche
await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
app/services/scraper_service.py
+9
View File
@@ -40,6 +40,7 @@ USER_AGENT_POOL = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
]
# Amazon blocking indicators
@@ -564,6 +565,14 @@ class ScraperService:
# Add random delay before navigation for Amazon
if is_amazon:
await asyncio.sleep(random.uniform(1.0, 3.0))
# Warmup: Visit home page first
try:
logger.info("Amazon Warmup: Visiting home page...")
await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
await asyncio.sleep(random.uniform(2.0, 4.0))
except Exception as e:
logger.warning(f"Amazon warmup failed: {e}")
logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
response = None