mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-17 04:08:05 +02:00
feat: Add BMStores product parser and a new Playwright-based scraping service with popup handling.
This commit is contained in:
Michael committed
1 parent
51ede950ff
commit
ef7cd7c1cf
5 files changed
+207
-87
No files matched your search
+33
-10
@@ -36,19 +36,42 @@ def get_amazon_proxies() -> list[dict]:
|
||||
})
|
||||
return proxies
|
||||
|
||||
# === USER AGENTS ===
|
||||
USER_AGENT_POOL = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
# === USER AGENTS & STEALTH ===
|
||||
USER_AGENT_DATA = [
|
||||
{
|
||||
"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"ch": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"platform": '"Windows"'
|
||||
},
|
||||
{
|
||||
"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"ch": '"Google Chrome";v="130", "Chromium";v="130", "Not_A Brand";v="99"',
|
||||
"platform": '"Windows"'
|
||||
},
|
||||
{
|
||||
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"ch": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"platform": '"macOS"'
|
||||
},
|
||||
{
|
||||
"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"ch": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"platform": '"Linux"'
|
||||
},
|
||||
{
|
||||
"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"ch": '"Microsoft Edge";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"platform": '"Windows"'
|
||||
}
|
||||
]
|
||||
|
||||
def get_random_stealth_config() -> dict:
|
||||
"""Returns a random User-Agent and its corresponding Client Hints"""
|
||||
return random.choice(USER_AGENT_DATA)
|
||||
|
||||
def get_random_user_agent() -> str:
|
||||
return random.choice(USER_AGENT_POOL)
|
||||
"""Legacy helper for backward compatibility"""
|
||||
return get_random_stealth_config()["ua"]
|
||||
|
||||
# === SITE CONFIGURATIONS ===
|
||||
SITE_CONFIGS = {
|
||||
|
||||
@@ -180,17 +180,21 @@ class AmazonScraperService:
|
||||
|
||||
@staticmethod
|
||||
async def _create_context(browser: Browser) -> BrowserContext:
|
||||
"""Create browser context with stealth settings"""
|
||||
"""Create browser context with dynamic stealth settings"""
|
||||
from app.core.search_config import get_random_stealth_config
|
||||
|
||||
stealth_config = get_random_stealth_config()
|
||||
ua = stealth_config["ua"]
|
||||
ch = stealth_config["ch"]
|
||||
platform = stealth_config["platform"]
|
||||
|
||||
logger.info(f"🎭 Using stealth profile: {ua[:50]}...")
|
||||
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
user_agent=ua,
|
||||
locale="fr-FR",
|
||||
timezone_id="Europe/Paris",
|
||||
# Additional stealth parameters
|
||||
extra_http_headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
@@ -198,22 +202,24 @@ class AmazonScraperService:
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Ch-Ua": ch,
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": platform,
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Referer": "https://www.google.fr/",
|
||||
},
|
||||
)
|
||||
|
||||
# Comprehensive stealth mode
|
||||
# Comprehensive stealth mode injector
|
||||
await context.add_init_script("""
|
||||
// Remove webdriver property
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined
|
||||
});
|
||||
// Reset webdriver
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
|
||||
// Add chrome property
|
||||
// Re-mock chrome object
|
||||
window.chrome = {
|
||||
runtime: {},
|
||||
loadTimes: function() {},
|
||||
@@ -221,43 +227,46 @@ class AmazonScraperService:
|
||||
app: {}
|
||||
};
|
||||
|
||||
// Override permissions
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (
|
||||
parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: Notification.permission }) :
|
||||
originalQuery(parameters)
|
||||
);
|
||||
|
||||
// Add plugins
|
||||
// Consistent plugins
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5]
|
||||
get: () => [
|
||||
{ name: 'PDF Viewer', filename: 'internal-pdf-viewer' },
|
||||
{ name: 'Chrome PDF Viewer', filename: 'internal-pdf-viewer' },
|
||||
{ name: 'Chromium PDF Viewer', filename: 'internal-pdf-viewer' }
|
||||
]
|
||||
});
|
||||
|
||||
// Add languages
|
||||
// Languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['fr-FR', 'fr', 'en-US', 'en']
|
||||
});
|
||||
|
||||
// Override platform
|
||||
Object.defineProperty(navigator, 'platform', {
|
||||
get: () => 'Win32'
|
||||
});
|
||||
|
||||
// Mock battery API
|
||||
Object.defineProperty(navigator, 'getBattery', {
|
||||
get: () => () => Promise.resolve({
|
||||
charging: true,
|
||||
chargingTime: 0,
|
||||
dischargingTime: Infinity,
|
||||
level: 1
|
||||
})
|
||||
});
|
||||
// Device Memory (randomized)
|
||||
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
||||
""")
|
||||
|
||||
await context.route("**/*", lambda route: route.continue_())
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
async def _simulate_human_behavior(page: Page):
|
||||
"""Perform subtle human-like interactions"""
|
||||
import random
|
||||
try:
|
||||
# Random mouse movements
|
||||
for _ in range(3):
|
||||
x = random.randint(100, 800)
|
||||
y = random.randint(100, 600)
|
||||
await page.mouse.move(x, y, steps=10)
|
||||
await asyncio.sleep(random.uniform(0.1, 0.3))
|
||||
|
||||
# Subtle scroll
|
||||
await page.evaluate("window.scrollBy(0, window.innerHeight / 4)")
|
||||
await asyncio.sleep(random.uniform(0.5, 1.0))
|
||||
await page.evaluate("window.scrollBy(0, -window.innerHeight / 5)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to simulate human behavior: {e}")
|
||||
|
||||
@staticmethod
|
||||
async def _handle_popups(page: Page):
|
||||
"""Close Amazon popups/cookies"""
|
||||
@@ -306,35 +315,44 @@ class AmazonScraperService:
|
||||
try:
|
||||
# CRITICAL: Load Amazon homepage FIRST in same context to establish session
|
||||
logger.info("🏠 Loading Amazon homepage to establish session/cookies...")
|
||||
await page.goto("https://www.amazon.fr", wait_until="domcontentloaded", timeout=30000)
|
||||
await page.goto("https://www.amazon.fr", wait_until="networkidle", timeout=30000)
|
||||
logger.info("✅ Homepage loaded")
|
||||
|
||||
# Handle homepage popups
|
||||
await cls._handle_popups(page)
|
||||
|
||||
# Simulate human behavior on homepage
|
||||
await cls._simulate_human_behavior(page)
|
||||
|
||||
# Small delay
|
||||
await page.wait_for_timeout(2000)
|
||||
await asyncio.sleep(random.uniform(1.0, 3.0))
|
||||
|
||||
# NOW navigate to search in SAME context (cookies preserved)
|
||||
logger.info(f"🔍 Navigating to search: {search_url}")
|
||||
await page.goto(search_url, wait_until="domcontentloaded", timeout=60000)
|
||||
logger.info("Page loaded (domcontentloaded)")
|
||||
|
||||
# Wait for network idle
|
||||
# Wait for content or block
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=10000)
|
||||
logger.info("Network idle reached")
|
||||
except PlaywrightTimeoutError:
|
||||
logger.info("Network idle timed out (non-critical)")
|
||||
|
||||
# Handle popups
|
||||
await cls._handle_popups(page)
|
||||
|
||||
# Wait a bit for content
|
||||
await page.wait_for_timeout(2000)
|
||||
await page.wait_for_selector('div[data-component-type="s-search-result"], .s-result-list', timeout=10000)
|
||||
logger.info("✅ Search results detected")
|
||||
await cls._simulate_human_behavior(page)
|
||||
except Exception:
|
||||
logger.warning("🕒 Search results not found immediately, checking for blocks...")
|
||||
|
||||
# Get HTML
|
||||
html_content = await page.content()
|
||||
|
||||
# Proactive block detection
|
||||
if "Type the characters you see in this image" in html_content or "Saisissez les caractères que vous voyez" in html_content:
|
||||
logger.error("🚫 CAPTCHA / Bot detection triggered")
|
||||
return []
|
||||
|
||||
if "Identifiez-vous" in html_content and "commander" not in html_content:
|
||||
logger.warning("⚠️ Redirected to login wall")
|
||||
# Try one more time with a different behavior or just fail
|
||||
return []
|
||||
|
||||
logger.info(f"✅ Page content extracted ({len(html_content)} bytes)")
|
||||
|
||||
if len(html_content) < 10000:
|
||||
|
||||
@@ -172,20 +172,31 @@ class BrowserlessService:
|
||||
|
||||
@staticmethod
|
||||
async def _create_context(browser: Browser, use_proxy: bool = False) -> BrowserContext:
|
||||
"""Create a new browser context with stealth settings."""
|
||||
user_agent = get_random_user_agent()
|
||||
"""Create a new browser context with dynamic stealth settings."""
|
||||
from app.core.search_config import get_random_stealth_config
|
||||
|
||||
stealth_config = get_random_stealth_config()
|
||||
ua = stealth_config["ua"]
|
||||
ch = stealth_config["ch"]
|
||||
platform = stealth_config["platform"]
|
||||
|
||||
logger.info(f"🎭 Using stealth profile: {ua[:50]}...")
|
||||
|
||||
options = {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
"user_agent": ua,
|
||||
"locale": "fr-FR",
|
||||
"timezone_id": "Europe/Paris",
|
||||
"java_script_enabled": True,
|
||||
"bypass_csp": True,
|
||||
"extra_http_headers": {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Sec-Ch-Ua-Platform": '"Windows"',
|
||||
"Sec-Ch-Ua": ch,
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": platform,
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Referer": "https://www.google.fr/",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -194,26 +205,35 @@ class BrowserlessService:
|
||||
|
||||
context = await browser.new_context(**options)
|
||||
|
||||
# Inject stealth scripts
|
||||
# Comprehensive stealth mode injector
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined
|
||||
// Reset webdriver
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
|
||||
// Re-mock chrome object
|
||||
window.chrome = {
|
||||
runtime: {},
|
||||
loadTimes: function() {},
|
||||
csi: function() {},
|
||||
app: {}
|
||||
};
|
||||
|
||||
// Consistent plugins
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [
|
||||
{ name: 'PDF Viewer', filename: 'internal-pdf-viewer' },
|
||||
{ name: 'Chrome PDF Viewer', filename: 'internal-pdf-viewer' },
|
||||
{ name: 'Chromium PDF Viewer', filename: 'internal-pdf-viewer' }
|
||||
]
|
||||
});
|
||||
|
||||
// Languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['fr-FR', 'fr', 'en-US', 'en']
|
||||
});
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5]
|
||||
});
|
||||
window.chrome = { runtime: {} };
|
||||
|
||||
// Mock permissions
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (
|
||||
parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: Notification.permission }) :
|
||||
originalQuery(parameters)
|
||||
);
|
||||
// Device Memory (randomized)
|
||||
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
||||
""")
|
||||
|
||||
await context.route("**/*", lambda route: route.continue_())
|
||||
@@ -517,28 +537,37 @@ class BrowserlessService:
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
# Random human-like lead-in delay
|
||||
import random
|
||||
await asyncio.sleep(random.uniform(0.5, 2.0))
|
||||
|
||||
await cls._navigate_and_wait(page, url, 30000)
|
||||
await cls._handle_popups(page)
|
||||
|
||||
# Check for Amazon Captcha / Login Wall
|
||||
# Check for Amazon Captcha / Login Wall / Blocking
|
||||
if "amazon" in url.lower():
|
||||
content_check = await page.content()
|
||||
if (
|
||||
is_blocked = (
|
||||
"Type the characters you see in this image" in content_check
|
||||
or "Saisissez les caractères que vous voyez" in content_check
|
||||
or "Sign in or create an account" in content_check
|
||||
or "Identifiez-vous ou créez un compte" in content_check
|
||||
):
|
||||
logger.warning(f"⚠️ Amazon Captcha/Login Wall detected (Attempt {attempt+1}/{retries})")
|
||||
or ("Sign in or create an account" in content_check and "orders" not in url.lower())
|
||||
or ("Identifiez-vous" in content_check and "commande" not in url.lower())
|
||||
or "api-services-support@amazon.com" in content_check
|
||||
or "service-unavailable" in content_check
|
||||
)
|
||||
|
||||
if is_blocked:
|
||||
logger.warning(f"⚠️ Amazon Blocking/Login Wall detected (Attempt {attempt+1}/{retries})")
|
||||
if attempt < retries - 1:
|
||||
await context.close()
|
||||
await asyncio.sleep(2 + attempt * 2) # Backoff
|
||||
# Exponential backoff with jitter
|
||||
await asyncio.sleep(2 * (attempt + 1) + random.uniform(0.5, 1.5))
|
||||
continue
|
||||
else:
|
||||
logger.error("❌ Amazon blocked all attempts")
|
||||
return "", ""
|
||||
|
||||
# Amazon-specific wait
|
||||
# Amazon-specific wait for price or content
|
||||
if "amazon" in url.lower() and "/dp/" in url:
|
||||
amazon_selectors = [".a-price .a-offscreen", "#corePriceDisplay_desktop_feature_div"]
|
||||
for selector in amazon_selectors:
|
||||
|
||||
@@ -56,3 +56,23 @@ class BMStoresParser(BaseParser):
|
||||
|
||||
logger.info(f"BMStoresParser found {len(results)} results")
|
||||
return results
|
||||
|
||||
def parse_product_details(self, html: str, product_url: str) -> dict:
|
||||
"""
|
||||
Extract price and stock from B&M product page
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
details = {"price": None, "in_stock": True}
|
||||
|
||||
# 1. Look for the main price (avoid suggested products prices)
|
||||
# B&M uses .current-price .price on detail pages
|
||||
price_el = soup.select_one(".product-prices .current-price .price, .product-price-and-shipping .price, [itemprop='price']")
|
||||
if price_el:
|
||||
details["price"] = self.parse_price_text(price_el.get_text())
|
||||
|
||||
# 2. Check stock
|
||||
stock_text = soup.get_text().lower()
|
||||
if "épuisé" in stock_text or "indisponible" in stock_text:
|
||||
details["in_stock"] = False
|
||||
|
||||
return details
|
||||
@@ -168,6 +168,36 @@ class ScraperService:
|
||||
|
||||
await ScraperService._handle_popups(page)
|
||||
|
||||
# FORCER LE MAGASIN NANCY POUR B&M STORES
|
||||
if "bmstores.fr" in url:
|
||||
try:
|
||||
logger.info("B&M Stores detected: checking store selection (Nancy)...")
|
||||
# Check if store is already selected or if modal is needed
|
||||
store_btn = page.locator(".mod-shops .btn-shop, .js-show-shops")
|
||||
if await store_btn.count() > 0:
|
||||
logger.info("Opening store selector...")
|
||||
await store_btn.first.click(timeout=2000)
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
# Type Nancy in the search input
|
||||
search_input = page.locator("#search_mag")
|
||||
if await search_input.count() > 0:
|
||||
await search_input.fill("Nancy")
|
||||
await page.keyboard.press("Enter")
|
||||
await page.wait_for_timeout(1500)
|
||||
|
||||
# Select the first Nancy store (Nancy Essey or Nancy Centre)
|
||||
select_btn = page.locator(".shop-list .btn-select-shop, button:has-text('Choisir ce magasin')")
|
||||
if await select_btn.count() > 0:
|
||||
logger.info("Selecting Nancy store...")
|
||||
await select_btn.first.click()
|
||||
await page.wait_for_timeout(2000)
|
||||
# Refresh page or wait for update
|
||||
await page.reload(wait_until="domcontentloaded")
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to force B&M store Nancy: {e}")
|
||||
|
||||
if selector:
|
||||
await ScraperService._wait_for_selector(page, selector)
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user