mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-12 01:39:25 +02:00
feat: Add core search configuration for site definitions, proxies, and user agents, and introduce a browserless service.
This commit is contained in:
Michael committed
1 parent
fa463ec50a
commit
6019a80de9
7 files changed
+559
-3358
No files matched your search
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
Search Configuration
|
||||
Centralizes all configuration for the search system:
|
||||
- Site definitions (selectors, URLs)
|
||||
- Proxy configurations
|
||||
- User Agents
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
# === BROWSERLESS CONFIGURATION ===
|
||||
BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
|
||||
DEBUG_DUMPS_DIR = "/app/debug_dumps"
|
||||
|
||||
# === PROXY CONFIGURATION ===
|
||||
# Format from user: ip:port:username:password
|
||||
AMAZON_PROXY_LIST_RAW = [
|
||||
"142.111.48.253:7030:jasuwwjr:elbsx170nmnl",
|
||||
"31.59.20.176:6754:jasuwwjr:elbsx170nmnl",
|
||||
"23.95.150.145:6114:jasuwwjr:elbsx170nmnl",
|
||||
"198.23.239.134:6540:jasuwwjr:elbsx170nmnl",
|
||||
"107.172.163.27:6543:jasuwwjr:elbsx170nmnl",
|
||||
"198.105.121.200:6462:jasuwwjr:elbsx170nmnl",
|
||||
"64.137.96.74:6641:jasuwwjr:elbsx170nmnl",
|
||||
"84.247.60.125:6095:jasuwwjr:elbsx170nmnl",
|
||||
"216.10.27.159:6837:jasuwwjr:elbsx170nmnl",
|
||||
"142.111.67.146:5611:jasuwwjr:elbsx170nmnl",
|
||||
]
|
||||
|
||||
def get_amazon_proxies() -> list[dict]:
|
||||
"""Convert raw proxy list to Playwright format"""
|
||||
proxies = []
|
||||
for proxy in AMAZON_PROXY_LIST_RAW:
|
||||
parts = proxy.split(":")
|
||||
if len(parts) == 4:
|
||||
proxies.append({
|
||||
"server": f"http://{parts[0]}:{parts[1]}",
|
||||
"username": parts[2],
|
||||
"password": parts[3]
|
||||
})
|
||||
return proxies
|
||||
|
||||
# === USER AGENTS ===
|
||||
USER_AGENT_POOL = [
|
||||
# Chrome Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
# Chrome Mac
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
# Firefox Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
# Edge Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
]
|
||||
|
||||
def get_random_user_agent() -> str:
|
||||
return random.choice(USER_AGENT_POOL)
|
||||
|
||||
# === SITE CONFIGURATIONS ===
|
||||
SITE_CONFIGS = {
|
||||
# === MAGASINS DISCOUNT ===
|
||||
"gifi.fr": {
|
||||
"name": "Gifi",
|
||||
"search_url": "https://www.gifi.fr/resultat-recherche?q={query}",
|
||||
"product_selector": "a.link, .product-item a.link, .product-tile a.link",
|
||||
"wait_selector": ".product-item, .product-tile, .products-grid",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"stokomani.fr": {
|
||||
"name": "Stokomani",
|
||||
"search_url": "https://www.stokomani.fr/recherche?q={query}",
|
||||
"product_selector": ".product-miniature a, .js-product-miniature a, a.product-thumbnail",
|
||||
"wait_selector": ".products, .product-miniature",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"bmstores.fr": {
|
||||
"name": "B&M",
|
||||
"search_url": "https://bmstores.fr/module/ambjolisearch/jolisearch?s={query}",
|
||||
"product_selector": "a.product-thumbnail[href*='/produits/']:not([href*='/c/'])",
|
||||
"wait_selector": ".products, .product-miniature",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"centrakor.com": {
|
||||
"name": "Centrakor",
|
||||
"search_url": "https://www.centrakor.com/catalogsearch/result/?q={query}",
|
||||
"product_selector": ".product-item a.product-item-link, .product-item-info a",
|
||||
"wait_selector": ".products-grid, .product-items",
|
||||
"category": "Déco & Maison",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"lincroyable.fr": {
|
||||
"name": "L'Incroyable",
|
||||
"search_url": "https://www.lincroyable.fr/recherche-query={query}/?",
|
||||
"product_selector": ".product-miniature a, .js-product-miniature a",
|
||||
"wait_selector": ".products, .product-miniature",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"action.com": {
|
||||
"name": "Action",
|
||||
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
||||
"product_selector": "a.product-card__link, .product-card a",
|
||||
"wait_selector": ".product-card, .search-results",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"lafoirfouille.fr": {
|
||||
"name": "La Foir'Fouille",
|
||||
"search_url": "https://www.lafoirfouille.fr/catalogsearch/result/?q={query}",
|
||||
"product_selector": ".product-item a.product-item-link, .product-item-info a",
|
||||
"wait_selector": ".products-grid, .product-items",
|
||||
"category": "Discount",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
# === GRANDES SURFACES ===
|
||||
"e.leclerc": {
|
||||
"name": "E.Leclerc",
|
||||
"search_url": "https://www.e.leclerc/recherche?q={query}",
|
||||
"product_selector": "a[href*='/fp/'][href*='-']:not([href*='promo'])",
|
||||
"wait_selector": "[data-testid='product-grid'], .search-results-list",
|
||||
"category": "Grande Surface",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"auchan.fr": {
|
||||
"name": "Auchan",
|
||||
"search_url": "https://www.auchan.fr/recherche?text={query}",
|
||||
"product_selector": "a[href*='/p/'][href$='.html']",
|
||||
"wait_selector": ".search-results, .product-grid",
|
||||
"category": "Grande Surface",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"carrefour.fr": {
|
||||
"name": "Carrefour",
|
||||
"search_url": "https://www.carrefour.fr/s?q={query}",
|
||||
"product_selector": "a[href*='/p/'][href$='.html']",
|
||||
"wait_selector": ".search-results, .product-grid",
|
||||
"category": "Grande Surface",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
# === E-COMMERCE GÉNÉRALISTE ===
|
||||
"amazon.fr": {
|
||||
"name": "Amazon France",
|
||||
"search_url": "https://www.amazon.fr/s?k={query}",
|
||||
"product_selector": "div[data-asin]:not([data-asin='']) h2 a",
|
||||
"wait_selector": "[data-component-type='s-search-result']",
|
||||
"category": "E-commerce",
|
||||
"requires_proxy": True,
|
||||
},
|
||||
"cdiscount.com": {
|
||||
"name": "Cdiscount",
|
||||
"search_url": "https://www.cdiscount.com/search/10/{query}.html",
|
||||
"product_selector": "a.prdtBImg[href*='/f-'][href*='.html']",
|
||||
"wait_selector": ".prdtBILDetails, .product-list",
|
||||
"category": "E-commerce",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
# === ÉLECTRONIQUE / HIGH-TECH ===
|
||||
"darty.com": {
|
||||
"name": "Darty",
|
||||
"search_url": "https://www.darty.com/nav/recherche?text={query}",
|
||||
"product_selector": "a[href*='/nav/achat/'][href*='.html']",
|
||||
"wait_selector": ".product-card, .product-list",
|
||||
"category": "Électronique",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"boulanger.com": {
|
||||
"name": "Boulanger",
|
||||
"search_url": "https://www.boulanger.com/resultats?tr={query}",
|
||||
"product_selector": "a[href*='/ref/'][href*='_']",
|
||||
"wait_selector": ".product-list, .product-card",
|
||||
"category": "Électronique",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
"fnac.com": {
|
||||
"name": "Fnac",
|
||||
"search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
|
||||
"product_selector": "a.Article-title[href*='/a']",
|
||||
"wait_selector": ".Article-item, .Article-list",
|
||||
"category": "Culture & Tech",
|
||||
"requires_proxy": False,
|
||||
},
|
||||
}
|
||||
|
||||
# === COOKIE BANNERS ===
|
||||
COOKIE_ACCEPT_SELECTORS = [
|
||||
"#sp-cc-accept", # Amazon
|
||||
"#onetrust-accept-btn-handler", # Many sites
|
||||
".onetrust-accept-btn-handler",
|
||||
"#popin_tc_privacy_button_2", # Auchan/Cdiscount
|
||||
".popin_tc_privacy_button",
|
||||
"#didomi-notice-agree-button", # Many French sites
|
||||
".didomi-continue-without-agreeing",
|
||||
"[data-testid='accept-cookies-button']", # Carrefour
|
||||
".bcom-consent-accept-all", # Boulanger
|
||||
"#cookieBanner-accept",
|
||||
".action-primary.action-accept", # Magento
|
||||
"#btn-cookie-allow",
|
||||
"button:has-text('Tout accepter')",
|
||||
"button:has-text('Accepter tout')",
|
||||
"button:has-text('Accepter')",
|
||||
"button:has-text('J\\'accepte')",
|
||||
]
|
||||
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
Browserless Service
|
||||
Manages connections to Browserless.io with advanced stealth capabilities.
|
||||
Handles:
|
||||
- Connection lifecycle
|
||||
- Stealth injection
|
||||
- Proxy rotation
|
||||
- Human behavior simulation
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from typing import Optional, Any
|
||||
|
||||
from playwright.async_api import async_playwright, Browser, BrowserContext, Page
|
||||
from app.core.search_config import (
|
||||
BROWSERLESS_URL,
|
||||
get_amazon_proxies,
|
||||
get_random_user_agent,
|
||||
COOKIE_ACCEPT_SELECTORS
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Stealth JS to inject
|
||||
STEALTH_JS = """
|
||||
// Advanced Stealth Mode
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
||||
delete Object.getPrototypeOf(navigator).webdriver;
|
||||
window.chrome = { runtime: {} };
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['fr-FR', 'fr', 'en-US', 'en'] });
|
||||
"""
|
||||
|
||||
class BrowserlessService:
|
||||
def __init__(self):
|
||||
self._playwright = None
|
||||
self._browser: Optional[Browser] = None
|
||||
self._proxies = get_amazon_proxies()
|
||||
|
||||
async def start(self):
|
||||
"""Initialize the browser connection"""
|
||||
if not self._playwright:
|
||||
self._playwright = await async_playwright().start()
|
||||
|
||||
if not self._browser or not self._browser.is_connected():
|
||||
try:
|
||||
logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
|
||||
# Use the stealth endpoint if possible, otherwise standard
|
||||
# Note: /chromium/stealth might need specific token or config,
|
||||
# falling back to standard connect_over_cdp which is safer for generic browserless
|
||||
self._browser = await self._playwright.chromium.connect_over_cdp(
|
||||
BROWSERLESS_URL,
|
||||
timeout=60000
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to Browserless: {e}")
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
"""Close resources"""
|
||||
if self._browser:
|
||||
await self._browser.close()
|
||||
if self._playwright:
|
||||
await self._playwright.stop()
|
||||
|
||||
async def get_context(self, use_proxy: bool = False) -> BrowserContext:
|
||||
"""Create a new context with stealth settings"""
|
||||
await self.start()
|
||||
|
||||
user_agent = get_random_user_agent()
|
||||
|
||||
options = {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"user_agent": user_agent,
|
||||
"locale": "fr-FR",
|
||||
"timezone_id": "Europe/Paris",
|
||||
"java_script_enabled": True,
|
||||
"bypass_csp": True,
|
||||
"extra_http_headers": {
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Sec-Ch-Ua-Platform": '"Windows"',
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
}
|
||||
|
||||
if use_proxy and self._proxies:
|
||||
proxy = random.choice(self._proxies)
|
||||
options["proxy"] = proxy
|
||||
logger.info(f"Using proxy: {proxy['server'].split('//')[1]}")
|
||||
|
||||
context = await self._browser.new_context(**options)
|
||||
|
||||
# Inject Stealth JS
|
||||
await context.add_init_script(STEALTH_JS)
|
||||
|
||||
# Block aggressive tracking but keep images for visual verification if needed
|
||||
# (Optimized for speed vs detection)
|
||||
await context.route("**/*", lambda route: route.continue_())
|
||||
|
||||
return context
|
||||
|
||||
async def simulate_human_behavior(self, page: Page):
|
||||
"""Simulate random mouse movements and scrolling"""
|
||||
try:
|
||||
# Mouse movements
|
||||
for _ in range(random.randint(2, 5)):
|
||||
await page.mouse.move(
|
||||
random.randint(100, 1800),
|
||||
random.randint(100, 900)
|
||||
)
|
||||
await asyncio.sleep(random.uniform(0.1, 0.3))
|
||||
|
||||
# Scrolling
|
||||
await page.evaluate("window.scrollBy(0, 300)")
|
||||
await asyncio.sleep(random.uniform(0.5, 1.0))
|
||||
await page.evaluate("window.scrollBy(0, -100)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def handle_cookie_banners(self, page: Page):
|
||||
"""Attempt to close cookie banners"""
|
||||
try:
|
||||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||||
if await page.locator(selector).first.is_visible(timeout=200):
|
||||
await page.locator(selector).first.click()
|
||||
logger.info(f"Closed cookie banner with {selector}")
|
||||
await asyncio.sleep(0.5)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def get_page_content(self, url: str, use_proxy: bool = False, wait_selector: str = None) -> tuple[str, str]:
|
||||
"""
|
||||
Fetch page content with full stealth lifecycle.
|
||||
Returns (html_content, screenshot_path)
|
||||
"""
|
||||
context = await self.get_context(use_proxy=use_proxy)
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
logger.info(f"Navigating to {url}")
|
||||
# Random delay before start
|
||||
await asyncio.sleep(random.uniform(0.5, 1.5))
|
||||
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
||||
|
||||
if response.status == 503:
|
||||
logger.warning(f"503 Detected on {url}")
|
||||
# Retry logic could be handled here or by caller,
|
||||
# for now we return what we have, caller decides
|
||||
|
||||
await self.handle_cookie_banners(page)
|
||||
await self.simulate_human_behavior(page)
|
||||
|
||||
if wait_selector:
|
||||
try:
|
||||
await page.wait_for_selector(wait_selector, timeout=10000)
|
||||
except Exception:
|
||||
logger.warning(f"Wait selector {wait_selector} timed out")
|
||||
|
||||
# Wait for network idle to ensure dynamic content loads
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
content = await page.content()
|
||||
return content, "" # Screenshot logic can be added if needed
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching {url}: {e}")
|
||||
return "", ""
|
||||
finally:
|
||||
await context.close()
|
||||
|
||||
# Global instance
|
||||
browserless_service = BrowserlessService()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,276 +0,0 @@
|
||||
"""
|
||||
Light Scraper Service - Scraping HTTP rapide sans navigateur
|
||||
Utilisé comme première tentative avant le fallback Browserless
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Headers pour simuler un navigateur
|
||||
DEFAULT_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Cache-Control": "max-age=0",
|
||||
}
|
||||
|
||||
# Patterns regex pour extraire les prix
|
||||
PRICE_PATTERNS = [
|
||||
# Format européen : 1 234,56 € ou 1234,56€
|
||||
r'(\d{1,3}(?:\s?\d{3})*[,\.]\d{2})\s*(?:€|EUR|euros?)',
|
||||
# Format : €1,234.56 ou € 1234.56
|
||||
r'(?:€|EUR)\s*(\d{1,3}(?:[,\s]?\d{3})*[,\.]\d{2})',
|
||||
# Prix simples : 12,99 € ou 12.99€
|
||||
r'(\d+[,\.]\d{2})\s*(?:€|EUR)',
|
||||
# data-price ou price attributes
|
||||
r'data-price["\']?\s*[:=]\s*["\']?(\d+[,\.]?\d*)',
|
||||
r'price["\']?\s*[:=]\s*["\']?(\d+[,\.]?\d*)',
|
||||
]
|
||||
|
||||
# Patterns pour le stock
|
||||
STOCK_POSITIVE_PATTERNS = [
|
||||
r'en\s*stock',
|
||||
r'disponible',
|
||||
r'in\s*stock',
|
||||
r'available',
|
||||
r'ajouter\s*au\s*panier',
|
||||
r'add\s*to\s*cart',
|
||||
r'acheter',
|
||||
r'livraison',
|
||||
]
|
||||
|
||||
STOCK_NEGATIVE_PATTERNS = [
|
||||
r'rupture',
|
||||
r'indisponible',
|
||||
r'out\s*of\s*stock',
|
||||
r'épuisé',
|
||||
r'sold\s*out',
|
||||
r'plus\s*disponible',
|
||||
r'non\s*disponible',
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LightScraperResult:
|
||||
"""Résultat du scraping léger"""
|
||||
|
||||
success: bool
|
||||
url: str
|
||||
title: str | None = None
|
||||
price: float | None = None
|
||||
currency: str = "EUR"
|
||||
in_stock: bool | None = None
|
||||
image_url: str | None = None
|
||||
error: str | None = None
|
||||
requires_js: bool = False # Indique si le site nécessite JS
|
||||
|
||||
|
||||
async def scrape_url(url: str, timeout: float = 10.0) -> LightScraperResult:
|
||||
"""
|
||||
Tente un scraping léger via HTTP simple.
|
||||
|
||||
Args:
|
||||
url: URL du produit à scraper
|
||||
timeout: Timeout en secondes
|
||||
|
||||
Returns:
|
||||
LightScraperResult avec les données extraites ou requires_js=True
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
follow_redirects=True,
|
||||
headers=DEFAULT_HEADERS,
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
html = response.text
|
||||
|
||||
# Vérifier si la page nécessite JavaScript
|
||||
if _requires_javascript(html):
|
||||
logger.debug(f"Page {url} nécessite JavaScript")
|
||||
return LightScraperResult(
|
||||
success=False,
|
||||
url=url,
|
||||
requires_js=True,
|
||||
error="Page requires JavaScript",
|
||||
)
|
||||
|
||||
# Extraire les données
|
||||
title = _extract_title(html)
|
||||
price = _extract_price(html)
|
||||
in_stock = _extract_stock_status(html)
|
||||
image_url = _extract_image(html, url)
|
||||
|
||||
# Si on n'a pas pu extraire le prix, fallback sur Browserless
|
||||
if price is None:
|
||||
logger.debug(f"Impossible d'extraire le prix de {url}")
|
||||
return LightScraperResult(
|
||||
success=False,
|
||||
url=url,
|
||||
title=title,
|
||||
requires_js=True,
|
||||
error="Could not extract price",
|
||||
)
|
||||
|
||||
logger.info(f"Scraping léger réussi pour {url}: {price}€")
|
||||
return LightScraperResult(
|
||||
success=True,
|
||||
url=url,
|
||||
title=title,
|
||||
price=price,
|
||||
in_stock=in_stock,
|
||||
image_url=image_url,
|
||||
)
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.warning(f"Timeout HTTP pour {url}")
|
||||
return LightScraperResult(
|
||||
success=False,
|
||||
url=url,
|
||||
error="HTTP timeout",
|
||||
requires_js=True,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(f"Erreur HTTP {e.response.status_code} pour {url}")
|
||||
return LightScraperResult(
|
||||
success=False,
|
||||
url=url,
|
||||
error=f"HTTP {e.response.status_code}",
|
||||
requires_js=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur scraping léger {url}: {e}")
|
||||
return LightScraperResult(
|
||||
success=False,
|
||||
url=url,
|
||||
error=str(e),
|
||||
requires_js=True,
|
||||
)
|
||||
|
||||
|
||||
def _requires_javascript(html: str) -> bool:
|
||||
"""Détecte si la page nécessite JavaScript pour afficher le contenu"""
|
||||
indicators = [
|
||||
"Please enable JavaScript",
|
||||
"JavaScript is required",
|
||||
"You need to enable JavaScript",
|
||||
"__NEXT_DATA__", # Next.js SSR mais peut nécessiter JS
|
||||
"window.__INITIAL_STATE__",
|
||||
"React.createElement",
|
||||
'<div id="root"></div>', # React SPA vide
|
||||
'<div id="app"></div>', # Vue SPA vide
|
||||
"Loading...",
|
||||
]
|
||||
|
||||
html_lower = html.lower()
|
||||
|
||||
# Vérifier si le body est presque vide
|
||||
body_match = re.search(r"<body[^>]*>(.*?)</body>", html, re.DOTALL | re.IGNORECASE)
|
||||
if body_match:
|
||||
body_content = body_match.group(1)
|
||||
# Retirer les scripts et styles
|
||||
body_clean = re.sub(r"<script[^>]*>.*?</script>", "", body_content, flags=re.DOTALL | re.IGNORECASE)
|
||||
body_clean = re.sub(r"<style[^>]*>.*?</style>", "", body_clean, flags=re.DOTALL | re.IGNORECASE)
|
||||
body_clean = re.sub(r"<[^>]+>", "", body_clean)
|
||||
body_clean = body_clean.strip()
|
||||
|
||||
if len(body_clean) < 500: # Page très peu de contenu texte
|
||||
return True
|
||||
|
||||
for indicator in indicators:
|
||||
if indicator.lower() in html_lower:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _extract_title(html: str) -> str | None:
|
||||
"""Extrait le titre de la page"""
|
||||
# Essayer og:title d'abord
|
||||
og_match = re.search(r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if og_match:
|
||||
return og_match.group(1).strip()
|
||||
|
||||
# Fallback sur <title>
|
||||
title_match = re.search(r"<title[^>]*>([^<]+)</title>", html, re.IGNORECASE)
|
||||
if title_match:
|
||||
title = title_match.group(1).strip()
|
||||
# Nettoyer les suffixes de site courants
|
||||
title = re.sub(r"\s*[-|]\s*(Amazon|Fnac|Cdiscount|Darty).*$", "", title, flags=re.IGNORECASE)
|
||||
return title
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_price(html: str) -> float | None:
|
||||
"""Extrait le prix de la page"""
|
||||
prices = []
|
||||
|
||||
for pattern in PRICE_PATTERNS:
|
||||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||||
for match in matches:
|
||||
try:
|
||||
# Normaliser le format
|
||||
price_str = match.replace(" ", "").replace(",", ".")
|
||||
price = float(price_str)
|
||||
if 0.01 < price < 100000: # Prix raisonnable
|
||||
prices.append(price)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not prices:
|
||||
return None
|
||||
|
||||
# Retourner le prix le plus fréquent ou le premier
|
||||
from collections import Counter
|
||||
|
||||
price_counts = Counter(prices)
|
||||
most_common = price_counts.most_common(1)
|
||||
if most_common:
|
||||
return most_common[0][0]
|
||||
|
||||
return prices[0]
|
||||
|
||||
|
||||
def _extract_stock_status(html: str) -> bool | None:
|
||||
"""Extrait le statut de stock"""
|
||||
html_lower = html.lower()
|
||||
|
||||
# Chercher les indicateurs négatifs d'abord
|
||||
for pattern in STOCK_NEGATIVE_PATTERNS:
|
||||
if re.search(pattern, html_lower):
|
||||
return False
|
||||
|
||||
# Chercher les indicateurs positifs
|
||||
for pattern in STOCK_POSITIVE_PATTERNS:
|
||||
if re.search(pattern, html_lower):
|
||||
return True
|
||||
|
||||
return None # Incertain
|
||||
|
||||
|
||||
def _extract_image(html: str, base_url: str) -> str | None:
|
||||
"""Extrait l'image principale du produit"""
|
||||
from urllib.parse import urljoin
|
||||
|
||||
# Essayer og:image d'abord
|
||||
og_match = re.search(r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if og_match:
|
||||
return urljoin(base_url, og_match.group(1))
|
||||
|
||||
# Essayer twitter:image
|
||||
twitter_match = re.search(r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if twitter_match:
|
||||
return urljoin(base_url, twitter_match.group(1))
|
||||
|
||||
return None
|
||||
@@ -1,948 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
|
||||
|
||||
# Proxy configuration for Amazon - 10 rotating proxies (same as direct_search_service)
|
||||
# Format from user: ip:port:username:password
|
||||
AMAZON_PROXY_LIST_RAW = [
|
||||
"142.111.48.253:7030:jasuwwjr:elbsx170nmnl",
|
||||
"31.59.20.176:6754:jasuwwjr:elbsx170nmnl",
|
||||
"23.95.150.145:6114:jasuwwjr:elbsx170nmnl",
|
||||
"198.23.239.134:6540:jasuwwjr:elbsx170nmnl",
|
||||
"107.172.163.27:6543:jasuwwjr:elbsx170nmnl",
|
||||
"198.105.121.200:6462:jasuwwjr:elbsx170nmnl",
|
||||
"64.137.96.74:6641:jasuwwjr:elbsx170nmnl",
|
||||
"84.247.60.125:6095:jasuwwjr:elbsx170nmnl",
|
||||
"216.10.27.159:6837:jasuwwjr:elbsx170nmnl",
|
||||
"142.111.67.146:5611:jasuwwjr:elbsx170nmnl",
|
||||
]
|
||||
|
||||
# Convert to Playwright proxy format: {"server": "http://ip:port", "username": "user", "password": "pass"}
|
||||
AMAZON_PROXY_LIST = []
|
||||
for proxy in AMAZON_PROXY_LIST_RAW:
|
||||
parts = proxy.split(":")
|
||||
if len(parts) == 4:
|
||||
AMAZON_PROXY_LIST.append({
|
||||
"server": f"http://{parts[0]}:{parts[1]}",
|
||||
"username": parts[2],
|
||||
"password": parts[3]
|
||||
})
|
||||
|
||||
# Log proxy initialization at startup
|
||||
logger.info(f"Amazon proxies initialized: {len(AMAZON_PROXY_LIST)} proxies available")
|
||||
|
||||
def _get_amazon_proxy() -> dict | None:
|
||||
"""Get a random proxy for Amazon requests from the rotating pool"""
|
||||
if AMAZON_PROXY_LIST:
|
||||
proxy = random.choice(AMAZON_PROXY_LIST)
|
||||
# Log only the IP part for security (hide credentials)
|
||||
ip_port = proxy["server"].replace("http://", "")
|
||||
logger.info(f"Using proxy: {ip_port}")
|
||||
return proxy
|
||||
logger.warning("No Amazon proxies available!")
|
||||
return None
|
||||
|
||||
# Nombre de tentatives pour le scraping
|
||||
MAX_RETRIES = 2
|
||||
RETRY_DELAY = 3 # secondes
|
||||
|
||||
# Amazon-specific configuration
|
||||
AMAZON_MAX_RETRIES = 4
|
||||
AMAZON_BASE_DELAY = 3.0
|
||||
|
||||
# Pool of realistic User-Agents for rotation
|
||||
USER_AGENT_POOL = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
# Amazon blocking indicators
|
||||
AMAZON_BLOCK_INDICATORS = [
|
||||
"Toutes nos excuses",
|
||||
"Sorry, something went wrong",
|
||||
"api-services-support@amazon.com",
|
||||
"Enter the characters you see below",
|
||||
"Saisissez les caractères",
|
||||
"Service Unavailable",
|
||||
"robot check",
|
||||
"automated access",
|
||||
]
|
||||
|
||||
# Stealth JavaScript to inject before page load to hide automation
|
||||
STEALTH_JS = """
|
||||
// ============================================
|
||||
// COMPREHENSIVE STEALTH MODE FOR AMAZON
|
||||
// ============================================
|
||||
|
||||
// 1. Override navigator.webdriver in multiple ways
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => false,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// Also delete from navigator prototype
|
||||
delete Object.getPrototypeOf(navigator).webdriver;
|
||||
|
||||
// Override the getAttribute method to hide webdriver attribute
|
||||
const originalGetAttribute = Element.prototype.getAttribute;
|
||||
Element.prototype.getAttribute = function(name) {
|
||||
if (name === 'webdriver') return null;
|
||||
return originalGetAttribute.call(this, name);
|
||||
};
|
||||
|
||||
// 2. Override navigator.plugins with realistic plugins
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => {
|
||||
const plugins = [
|
||||
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
|
||||
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 },
|
||||
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '', length: 2 },
|
||||
{ name: 'Chromium PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 },
|
||||
{ name: 'Chromium PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 }
|
||||
];
|
||||
plugins.item = (i) => plugins[i] || null;
|
||||
plugins.namedItem = (name) => plugins.find(p => p.name === name) || null;
|
||||
plugins.refresh = () => {};
|
||||
plugins.length = plugins.length;
|
||||
return plugins;
|
||||
},
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 3. Override navigator.mimeTypes
|
||||
Object.defineProperty(navigator, 'mimeTypes', {
|
||||
get: () => {
|
||||
const mimeTypes = [
|
||||
{ type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
|
||||
{ type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' }
|
||||
];
|
||||
mimeTypes.item = (i) => mimeTypes[i] || null;
|
||||
mimeTypes.namedItem = (name) => mimeTypes.find(m => m.type === name) || null;
|
||||
mimeTypes.length = mimeTypes.length;
|
||||
return mimeTypes;
|
||||
},
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 4. Override navigator.languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['fr-FR', 'fr', 'en-US', 'en'],
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 5. Override permissions API
|
||||
if (window.navigator.permissions) {
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => {
|
||||
if (parameters.name === 'notifications') {
|
||||
return Promise.resolve({ state: Notification.permission, onchange: null });
|
||||
}
|
||||
return originalQuery.call(window.navigator.permissions, parameters);
|
||||
};
|
||||
}
|
||||
|
||||
// 6. Hide automation-related properties (Chromium DevTools Protocol)
|
||||
const propsToDelete = [
|
||||
'cdc_adoQpoasnfa76pfcZLmcfl_Array',
|
||||
'cdc_adoQpoasnfa76pfcZLmcfl_Promise',
|
||||
'cdc_adoQpoasnfa76pfcZLmcfl_Symbol',
|
||||
'__webdriver_evaluate',
|
||||
'__selenium_evaluate',
|
||||
'__webdriver_script_function',
|
||||
'__webdriver_script_func',
|
||||
'__webdriver_script_fn',
|
||||
'__fxdriver_evaluate',
|
||||
'__driver_unwrapped',
|
||||
'__webdriver_unwrapped',
|
||||
'__driver_evaluate',
|
||||
'__selenium_unwrapped',
|
||||
'__fxdriver_unwrapped',
|
||||
'_Selenium_IDE_Recorder',
|
||||
'_selenium',
|
||||
'calledSelenium',
|
||||
'$chrome_asyncScriptInfo',
|
||||
'$cdc_asdjflasutopfhvcZLmcfl_',
|
||||
'$wdc_'
|
||||
];
|
||||
propsToDelete.forEach(prop => {
|
||||
try { delete window[prop]; } catch(e) {}
|
||||
});
|
||||
|
||||
// 7. Override chrome object with realistic properties
|
||||
window.chrome = {
|
||||
app: {
|
||||
isInstalled: false,
|
||||
InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
|
||||
RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
|
||||
},
|
||||
runtime: {
|
||||
OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
|
||||
OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
|
||||
PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
|
||||
PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
|
||||
PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
|
||||
RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
|
||||
connect: function() { return { onDisconnect: { addListener: function() {} }, onMessage: { addListener: function() {} }, postMessage: function() {} }; },
|
||||
sendMessage: function() {}
|
||||
},
|
||||
csi: function() { return {}; },
|
||||
loadTimes: function() { return { requestTime: Date.now() / 1000, startLoadTime: Date.now() / 1000, firstPaintAfterLoadTime: 0, firstPaintTime: Date.now() / 1000, navigationType: 'navigate' }; }
|
||||
};
|
||||
|
||||
// 8. WebGL fingerprinting protection
|
||||
const getParameterProxyHandler = {
|
||||
apply: function(target, ctx, args) {
|
||||
if (args[0] === 37445) return 'Intel Inc.';
|
||||
if (args[0] === 37446) return 'Intel Iris OpenGL Engine';
|
||||
if (args[0] === 7937) return 'WebKit';
|
||||
if (args[0] === 7936) return 'WebKit WebGL';
|
||||
return Reflect.apply(target, ctx, args);
|
||||
}
|
||||
};
|
||||
try {
|
||||
const canvas = document.createElement('canvas');
|
||||
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
|
||||
if (gl) {
|
||||
const getParameter = gl.getParameter.bind(gl);
|
||||
gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
|
||||
}
|
||||
const gl2 = canvas.getContext('webgl2');
|
||||
if (gl2) {
|
||||
const getParameter2 = gl2.getParameter.bind(gl2);
|
||||
gl2.getParameter = new Proxy(getParameter2, getParameterProxyHandler);
|
||||
}
|
||||
} catch(e) {}
|
||||
|
||||
// 9. Canvas fingerprinting protection
|
||||
const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
||||
HTMLCanvasElement.prototype.toDataURL = function(type) {
|
||||
if (type === 'image/png' && this.width === 220 && this.height === 30) {
|
||||
const context = this.getContext('2d');
|
||||
if (context) {
|
||||
const imageData = context.getImageData(0, 0, this.width, this.height);
|
||||
for (let i = 0; i < imageData.data.length; i += 4) {
|
||||
imageData.data[i] = imageData.data[i] ^ (Math.random() * 2);
|
||||
}
|
||||
context.putImageData(imageData, 0, 0);
|
||||
}
|
||||
}
|
||||
return originalToDataURL.apply(this, arguments);
|
||||
};
|
||||
|
||||
// 10. AudioContext fingerprinting protection
|
||||
if (window.AudioContext || window.webkitAudioContext) {
|
||||
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
||||
const originalCreateOscillator = AudioContext.prototype.createOscillator;
|
||||
AudioContext.prototype.createOscillator = function() {
|
||||
const oscillator = originalCreateOscillator.apply(this, arguments);
|
||||
oscillator.frequency.value = oscillator.frequency.value + (Math.random() * 0.0001);
|
||||
return oscillator;
|
||||
};
|
||||
}
|
||||
|
||||
// 11. Override connection properties
|
||||
Object.defineProperty(navigator, 'connection', {
|
||||
get: () => ({
|
||||
effectiveType: '4g',
|
||||
rtt: 50 + Math.floor(Math.random() * 50),
|
||||
downlink: 10 + Math.random() * 5,
|
||||
saveData: false
|
||||
}),
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 12. Override hardware properties
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
||||
get: () => 8,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
Object.defineProperty(navigator, 'deviceMemory', {
|
||||
get: () => 8,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 13. Override screen properties
|
||||
Object.defineProperty(screen, 'colorDepth', {
|
||||
get: () => 24,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
Object.defineProperty(screen, 'pixelDepth', {
|
||||
get: () => 24,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 14. Spoof Notification API
|
||||
if (window.Notification) {
|
||||
Object.defineProperty(Notification, 'permission', {
|
||||
get: () => 'default',
|
||||
configurable: true
|
||||
});
|
||||
}
|
||||
|
||||
// 15. Override iframe contentWindow checks
|
||||
const originalContentWindow = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow');
|
||||
Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', {
|
||||
get: function() {
|
||||
const win = originalContentWindow.get.call(this);
|
||||
if (win) {
|
||||
try {
|
||||
Object.defineProperty(win.navigator, 'webdriver', {
|
||||
get: () => false,
|
||||
configurable: true
|
||||
});
|
||||
} catch(e) {}
|
||||
}
|
||||
return win;
|
||||
}
|
||||
});
|
||||
|
||||
// 16. Mock Battery API
|
||||
if (navigator.getBattery) {
|
||||
navigator.getBattery = () => Promise.resolve({
|
||||
charging: true,
|
||||
chargingTime: 0,
|
||||
dischargingTime: Infinity,
|
||||
level: 1,
|
||||
addEventListener: () => {},
|
||||
removeEventListener: () => {}
|
||||
});
|
||||
}
|
||||
|
||||
// 17. Hide Playwright/Puppeteer specific objects
|
||||
delete window.__playwright;
|
||||
delete window.__pw_manual;
|
||||
delete window.__PW_inspect;
|
||||
|
||||
console.log('Advanced stealth mode activated');
|
||||
"""
|
||||
|
||||
|
||||
def _get_random_user_agent() -> str:
|
||||
"""Get a random User-Agent from the pool"""
|
||||
return random.choice(USER_AGENT_POOL)
|
||||
|
||||
|
||||
def _is_amazon_url(url: str) -> bool:
|
||||
"""Check if URL is an Amazon URL"""
|
||||
return "amazon" in url.lower()
|
||||
|
||||
|
||||
def _is_amazon_blocked(html_content: str) -> bool:
|
||||
"""Check if Amazon has blocked the request"""
|
||||
html_lower = html_content.lower()
|
||||
for indicator in AMAZON_BLOCK_INDICATORS:
|
||||
if indicator.lower() in html_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def simplify_url(url: str) -> str:
|
||||
"""
|
||||
Simplifie les URLs en supprimant les paramètres de tracking.
|
||||
Particulièrement utile pour Amazon qui a des URLs très longues.
|
||||
|
||||
Exemples:
|
||||
- Amazon: https://www.amazon.fr/dp/B0CFYHHPPV?ref=... -> https://www.amazon.fr/dp/B0CFYHHPPV
|
||||
- Autres: garde l'URL originale si pas de simplification possible
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
|
||||
# Simplification pour Amazon
|
||||
if "amazon" in hostname:
|
||||
# Extraire l'ASIN (identifiant produit Amazon)
|
||||
# Format: /dp/ASIN ou /gp/product/ASIN
|
||||
asin_match = re.search(r"/(?:dp|gp/product)/([A-Z0-9]{10})", url)
|
||||
if asin_match:
|
||||
asin = asin_match.group(1)
|
||||
# Reconstruire une URL simple
|
||||
simplified = f"{parsed.scheme}://{parsed.netloc}/dp/{asin}"
|
||||
logger.info(f"URL Amazon simplifiée: {url[:80]}... -> {simplified}")
|
||||
return simplified
|
||||
|
||||
# Pour les autres sites, supprimer les paramètres de tracking courants
|
||||
# mais garder les paramètres essentiels
|
||||
tracking_params = [
|
||||
"utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term",
|
||||
"ref", "ref_", "tag", "linkCode", "linkId", "camp", "creative",
|
||||
"crid", "dib", "dib_tag", "qid", "sprefix", "sr", "keywords",
|
||||
"fbclid", "gclid", "msclkid"
|
||||
]
|
||||
|
||||
if parsed.query:
|
||||
# Garder seulement les paramètres non-tracking
|
||||
from urllib.parse import parse_qs, urlencode
|
||||
params = parse_qs(parsed.query)
|
||||
filtered_params = {
|
||||
k: v[0] for k, v in params.items()
|
||||
if k.lower() not in tracking_params
|
||||
}
|
||||
new_query = urlencode(filtered_params) if filtered_params else ""
|
||||
simplified = urlunparse((
|
||||
parsed.scheme,
|
||||
parsed.netloc,
|
||||
parsed.path,
|
||||
parsed.params,
|
||||
new_query,
|
||||
"" # fragment
|
||||
))
|
||||
if simplified != url:
|
||||
logger.info(f"URL simplifiée: {url[:60]}... -> {simplified[:60]}...")
|
||||
return simplified
|
||||
|
||||
return url
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Erreur lors de la simplification de l'URL: {e}")
|
||||
return url
|
||||
|
||||
|
||||
class ScraperService:
|
||||
@classmethod
|
||||
async def scrape_item(
|
||||
cls,
|
||||
url: str,
|
||||
selector: str | None = None,
|
||||
item_id: int | None = None,
|
||||
smart_scroll: bool = False,
|
||||
scroll_pixels: int = 350,
|
||||
text_length: int = 0,
|
||||
timeout: int = 90000,
|
||||
browser=None, # Support external browser instance
|
||||
) -> tuple[str | None, str, bool]:
|
||||
"""
|
||||
Scrape a product page with retries and rotation.
|
||||
Manages the browser lifecycle to reuse connections across retries.
|
||||
If 'browser' is provided, it uses that instance and DOES NOT close it.
|
||||
"""
|
||||
is_amazon = _is_amazon_url(url)
|
||||
max_retries = AMAZON_MAX_RETRIES if is_amazon else MAX_RETRIES
|
||||
|
||||
playwright_manager = None
|
||||
local_browser = None
|
||||
|
||||
# Determine if we are managing the browser or using an external one
|
||||
using_external_browser = browser is not None
|
||||
current_browser = browser
|
||||
|
||||
try:
|
||||
# Initial browser creation if not provided
|
||||
if not current_browser:
|
||||
try:
|
||||
playwright_manager = async_playwright()
|
||||
p = await playwright_manager.start()
|
||||
logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
|
||||
local_browser = await p.chromium.connect_over_cdp(
|
||||
BROWSERLESS_URL,
|
||||
timeout=60000,
|
||||
)
|
||||
current_browser = local_browser
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize browser: {e}")
|
||||
return None, "", False
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
# Check if browser is still connected
|
||||
if not current_browser or not current_browser.is_connected():
|
||||
if using_external_browser:
|
||||
logger.warning("External browser disconnected! Cannot proceed with this browser.")
|
||||
# We cannot reconnect an external browser here as we don't own the playwright instance
|
||||
# We could try to fall back to a local browser, but for now let's just fail
|
||||
return None, "", False
|
||||
|
||||
logger.warning("Browser disconnected, reconnecting...")
|
||||
try:
|
||||
if current_browser:
|
||||
try:
|
||||
await current_browser.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Reconnect local browser
|
||||
current_browser = await p.chromium.connect_over_cdp(
|
||||
BROWSERLESS_URL,
|
||||
timeout=60000,
|
||||
)
|
||||
local_browser = current_browser
|
||||
except Exception as re_e:
|
||||
logger.error(f"Failed to reconnect browser: {re_e}")
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
|
||||
logger.info(f"Scraping attempt {attempt + 1}/{max_retries + 1} for {url}")
|
||||
|
||||
screenshot, text, is_available = await cls._do_scrape(
|
||||
url,
|
||||
selector,
|
||||
item_id,
|
||||
smart_scroll,
|
||||
scroll_pixels,
|
||||
text_length,
|
||||
timeout,
|
||||
browser=current_browser,
|
||||
is_amazon=is_amazon,
|
||||
attempt=attempt,
|
||||
max_retries=max_retries,
|
||||
)
|
||||
|
||||
if screenshot or (text and len(text) > 0):
|
||||
return screenshot, text, is_available
|
||||
|
||||
logger.warning(f"Tentative {attempt + 1} échouée pour {url}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in scrape_item attempt {attempt + 1}: {e}")
|
||||
# If it's a critical browser error
|
||||
if "Target page, context or browser has been closed" in str(e) or "Connection closed" in str(e):
|
||||
if using_external_browser:
|
||||
logger.warning("Critical error with external browser.")
|
||||
# Stop retrying with this browser
|
||||
return None, "", False
|
||||
|
||||
logger.warning("Critical browser error detected, forcing reconnection.")
|
||||
try:
|
||||
if current_browser:
|
||||
await current_browser.close()
|
||||
except:
|
||||
pass
|
||||
current_browser = None
|
||||
|
||||
if attempt < max_retries:
|
||||
delay = AMAZON_BASE_DELAY * (attempt + 1) + random.uniform(1, 3) if is_amazon else RETRY_DELAY
|
||||
logger.info(f"Tentative {attempt + 2}/{max_retries + 1} pour {url} (délai: {delay:.1f}s)")
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
logger.error(f"Toutes les tentatives ont échoué pour {url}")
|
||||
return None, "", False
|
||||
|
||||
finally:
|
||||
# Only close resources we created
|
||||
if local_browser:
|
||||
try:
|
||||
if local_browser.is_connected():
|
||||
await local_browser.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error closing local browser in scrape_item: {e}")
|
||||
|
||||
if playwright_manager:
|
||||
try:
|
||||
await playwright_manager.stop()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error stopping playwright manager in scrape_item: {e}")
|
||||
|
||||
@classmethod
|
||||
async def _do_scrape(
|
||||
cls,
|
||||
url: str,
|
||||
selector: str | None = None,
|
||||
item_id: int | None = None,
|
||||
smart_scroll: bool = False,
|
||||
scroll_pixels: int = 350,
|
||||
text_length: int = 0,
|
||||
timeout: int = 90000,
|
||||
browser=None,
|
||||
is_amazon: bool = False,
|
||||
attempt: int = 0,
|
||||
max_retries: int = 1,
|
||||
) -> tuple[str | None, str, bool]:
|
||||
"""Exécute le scraping réel (appelé par scrape_item avec retries)."""
|
||||
|
||||
# Si un navigateur est fourni, on l'utilise directement sans async_playwright context manager
|
||||
# sinon on crée tout de zéro
|
||||
playwright_manager = None
|
||||
local_browser = None
|
||||
|
||||
try:
|
||||
if browser:
|
||||
# Utiliser le navigateur partagé
|
||||
current_browser = browser
|
||||
else:
|
||||
# Créer un nouveau navigateur
|
||||
playwright_manager = async_playwright()
|
||||
p = await playwright_manager.start()
|
||||
logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
|
||||
current_browser = await p.chromium.connect_over_cdp(
|
||||
BROWSERLESS_URL,
|
||||
timeout=60000,
|
||||
)
|
||||
local_browser = current_browser
|
||||
|
||||
context = None
|
||||
page = None
|
||||
is_available = True # Par défaut, le produit est disponible
|
||||
|
||||
# Use random User-Agent for anti-detection (especially for Amazon)
|
||||
current_user_agent = _get_random_user_agent()
|
||||
|
||||
# Enhanced headers for stealth
|
||||
extra_headers = {
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": '"Windows"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
# Add referer for Amazon
|
||||
if is_amazon:
|
||||
extra_headers["Referer"] = "https://www.google.fr/"
|
||||
|
||||
# Gestion intelligente du Proxy :
|
||||
# - Tentatives 0 à MAX-1 : Utiliser un Proxy (si dispo)
|
||||
# - Dernière tentative (max_retries) : Essayer SANS Proxy (Direct IP) en dernier recours
|
||||
# (car l'utilisateur signale que l'accès direct marche parfois mieux pour les pages produits)
|
||||
use_proxy = is_amazon and (attempt < max_retries)
|
||||
|
||||
if not use_proxy and is_amazon and attempt > 0:
|
||||
logger.info(f"Tentative {attempt + 1}/{max_retries + 1} : Passage en mode DIRECT (Sans Proxy) pour tenter de contourner le blocage proxy")
|
||||
|
||||
# Prepare context options
|
||||
context_options = {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"user_agent": current_user_agent,
|
||||
"locale": "fr-FR",
|
||||
"timezone_id": "Europe/Paris",
|
||||
"extra_http_headers": extra_headers,
|
||||
"java_script_enabled": True,
|
||||
"bypass_csp": True,
|
||||
}
|
||||
|
||||
# Add proxy for Amazon if configured AND enabled for this attempt
|
||||
if use_proxy:
|
||||
proxy_config = _get_amazon_proxy()
|
||||
if proxy_config:
|
||||
# proxy_config is already a dict with {"server": "...", "username": "...", "password": "..."}
|
||||
context_options["proxy"] = proxy_config
|
||||
|
||||
# Création du contexte
|
||||
context = await current_browser.new_context(**context_options)
|
||||
|
||||
# Inject stealth script before any page loads (especially for Amazon)
|
||||
if is_amazon:
|
||||
await context.add_init_script(STEALTH_JS)
|
||||
logger.debug("Stealth JS injected for Amazon product page")
|
||||
|
||||
# Stealth mode / Ad blocking attempts - don't block images for Amazon
|
||||
if is_amazon:
|
||||
# For Amazon, only block tracking - keep images to avoid detection
|
||||
await context.route("**/analytics*", lambda route: route.abort())
|
||||
await context.route("**/tracking*", lambda route: route.abort())
|
||||
else:
|
||||
await context.route("**/*", lambda route: route.continue_())
|
||||
|
||||
page = await context.new_page()
|
||||
|
||||
# Add random delay before navigation for Amazon
|
||||
if is_amazon:
|
||||
await asyncio.sleep(random.uniform(1.0, 3.0))
|
||||
|
||||
# Warmup: Visit home page first
|
||||
try:
|
||||
logger.info("Amazon Warmup: Visiting home page...")
|
||||
await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000)
|
||||
await asyncio.sleep(random.uniform(2.0, 4.0))
|
||||
except Exception as e:
|
||||
logger.warning(f"Amazon warmup failed: {e}")
|
||||
|
||||
logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
|
||||
response = None
|
||||
amazon_blocked = False
|
||||
try:
|
||||
# First wait for domcontentloaded - this is the minimum we need
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
||||
|
||||
# Check HTTP status code
|
||||
if response:
|
||||
status = response.status
|
||||
logger.info(f"HTTP Status: {status}")
|
||||
|
||||
# For Amazon, 503 is usually a block, not product unavailability
|
||||
if is_amazon and status == 503:
|
||||
logger.warning(f"Amazon returned 503 - likely blocking")
|
||||
amazon_blocked = True
|
||||
# Detect unavailable products by HTTP status (non-Amazon)
|
||||
elif status in [404, 410, 451]:
|
||||
logger.warning(f"Product unavailable - HTTP {status}")
|
||||
is_available = False
|
||||
elif status == 503 and not is_amazon:
|
||||
logger.warning(f"Product unavailable - HTTP {status}")
|
||||
is_available = False
|
||||
elif status >= 400:
|
||||
logger.warning(f"HTTP error {status}, marking as potentially unavailable")
|
||||
is_available = False
|
||||
|
||||
logger.info(f"Page loaded (domcontentloaded): {url}")
|
||||
|
||||
# Then try to wait for networkidle, but don't fail if it times out
|
||||
# This helps with heavy pages that never fully settle
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||
logger.info("Network idle reached")
|
||||
except PlaywrightTimeoutError:
|
||||
logger.info("Network idle timed out (non-critical), proceeding...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error navigating to {url}: {e}")
|
||||
# If navigation fails completely, mark as unavailable
|
||||
is_available = False
|
||||
# Try to take screenshot anyway if page partially loaded
|
||||
pass
|
||||
|
||||
# Wait a bit for dynamic content if needed - longer for Amazon
|
||||
wait_time = random.uniform(2.0, 4.0) if is_amazon else 2.0
|
||||
await page.wait_for_timeout(int(wait_time * 1000))
|
||||
|
||||
# Check for Amazon blocking in page content
|
||||
if is_amazon:
|
||||
try:
|
||||
page_html = await page.content()
|
||||
if _is_amazon_blocked(page_html):
|
||||
logger.warning(f"Amazon blocking detected in page content for {url}")
|
||||
amazon_blocked = True
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not check Amazon blocking: {e}")
|
||||
|
||||
# If Amazon blocked us, return None to trigger retry
|
||||
if amazon_blocked:
|
||||
logger.warning(f"Amazon blocked - will retry with different User-Agent")
|
||||
return None, "", True # Return None screenshot to trigger retry
|
||||
|
||||
# Try to close common popups and cookie banners
|
||||
logger.info("Attempting to close popups and cookie banners...")
|
||||
popup_selectors = [
|
||||
# === SITES SPÉCIFIQUES FRANÇAIS ===
|
||||
# Amazon France
|
||||
"#sp-cc-accept",
|
||||
"#sp-cc-rejectall-link",
|
||||
"input[data-action-type='DISMISS']",
|
||||
"#a-popover-content-1 button",
|
||||
"[data-action='a-popover-close']",
|
||||
# E.Leclerc, Darty, Fnac
|
||||
"#onetrust-accept-btn-handler",
|
||||
".onetrust-accept-btn-handler",
|
||||
# Auchan
|
||||
"#popin_tc_privacy_button_2",
|
||||
".popin_tc_privacy_button",
|
||||
"#didomi-notice-agree-button",
|
||||
# Carrefour
|
||||
"[data-testid='accept-cookies-button']",
|
||||
# Cdiscount
|
||||
"#footer_tc_privacy_button_2",
|
||||
".privacy_prompt_accept",
|
||||
# Boulanger
|
||||
".bcom-consent-accept-all",
|
||||
"#cookieBanner-accept",
|
||||
# Gifi, Centrakor, La Foir'Fouille (Magento)
|
||||
".action-primary.action-accept",
|
||||
"#btn-cookie-allow",
|
||||
# PrestaShop (Stokomani, B&M, etc.)
|
||||
".btn-primary[data-dismiss='modal']",
|
||||
"#gdpr_consent_agree",
|
||||
|
||||
# === CMP (Consent Management Platforms) ===
|
||||
".didomi-continue-without-agreeing",
|
||||
"#tarteaucitronPersonalize2",
|
||||
".tarteaucitronAllow",
|
||||
"#tarteaucitronAllDenied2",
|
||||
".cc-btn.cc-allow",
|
||||
".cky-btn-accept",
|
||||
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
|
||||
"#CybotCookiebotDialogBodyButtonAccept",
|
||||
".qc-cmp2-summary-buttons button:first-child",
|
||||
"#axeptio_btn_acceptAll",
|
||||
".axeptio_acceptAll",
|
||||
|
||||
# === TEXTE FRANÇAIS ===
|
||||
"button:has-text('Tout accepter')",
|
||||
"button:has-text('Accepter tout')",
|
||||
"button:has-text('Accepter et fermer')",
|
||||
"button:has-text('Accepter les cookies')",
|
||||
"button:has-text('Accepter')",
|
||||
"button:has-text('J\\'accepte')",
|
||||
"button:has-text('OK')",
|
||||
"button:has-text('Continuer')",
|
||||
"a:has-text('Tout accepter')",
|
||||
"a:has-text('Accepter')",
|
||||
|
||||
# === POPUPS GÉNÉRAUX ===
|
||||
"button[aria-label='Close']",
|
||||
"button[aria-label='close']",
|
||||
"button[aria-label='Fermer']",
|
||||
".close-button",
|
||||
".modal-close",
|
||||
"svg[data-name='Close']",
|
||||
"[class*='popup'] button",
|
||||
"[class*='modal'] button",
|
||||
"button:has-text('No, thanks')",
|
||||
"button:has-text('No thanks')",
|
||||
"button:has-text('Accept')",
|
||||
"a:has-text('No, thanks')",
|
||||
"div[role='dialog'] button[aria-label='Close']",
|
||||
]
|
||||
|
||||
for popup_selector in popup_selectors:
|
||||
try:
|
||||
if await page.locator(popup_selector).count() > 0:
|
||||
logger.info(f"Found popup close button: {popup_selector}")
|
||||
# Try to click it. If it fails, catch and continue
|
||||
await page.locator(popup_selector).first.click(timeout=2000)
|
||||
await page.wait_for_timeout(1000) # Wait for animation
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not close popup with selector {popup_selector}: {e}")
|
||||
|
||||
# Also try pressing Escape
|
||||
try:
|
||||
await page.keyboard.press("Escape")
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not press Escape key: {e}")
|
||||
|
||||
if selector:
|
||||
try:
|
||||
logger.info(f"Waiting for selector: {selector}")
|
||||
await page.wait_for_selector(selector, timeout=5000)
|
||||
# Scroll to element
|
||||
element = page.locator(selector).first
|
||||
await element.scroll_into_view_if_needed()
|
||||
logger.info(f"Scrolled to selector: {selector}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Selector {selector} not found or timed out: {e}")
|
||||
else:
|
||||
# Auto-detect price if no selector
|
||||
logger.info("No selector provided. Attempting to find price element...")
|
||||
try:
|
||||
# Look for common price patterns
|
||||
price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/")
|
||||
if await price_locator.count() > 0:
|
||||
# Pick the first one that looks visible and reasonable size
|
||||
# This is heuristic
|
||||
await price_locator.first.scroll_into_view_if_needed()
|
||||
logger.info("Scrolled to potential price element")
|
||||
except Exception as e:
|
||||
logger.warning(f"Auto-price detection failed: {e}")
|
||||
|
||||
# Smart Scroll
|
||||
if smart_scroll:
|
||||
logger.info(f"Performing smart scroll ({scroll_pixels}px)...")
|
||||
try:
|
||||
await page.evaluate(f"window.scrollBy(0, {scroll_pixels})")
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception as e:
|
||||
logger.warning(f"Smart scroll failed: {e}")
|
||||
|
||||
# IMPORTANT: Prendre le screenshot AVANT l'extraction de texte
|
||||
# pour éviter de perdre le screenshot si le navigateur se ferme
|
||||
screenshot_dir = "screenshots"
|
||||
os.makedirs(screenshot_dir, exist_ok=True)
|
||||
if item_id:
|
||||
filename = f"{screenshot_dir}/item_{item_id}.png"
|
||||
else:
|
||||
url_part = url.split("//")[-1].replace("/", "_")[:50] # Limiter la longueur
|
||||
timestamp = int(asyncio.get_event_loop().time())
|
||||
filename = f"{screenshot_dir}/{url_part}_{timestamp}.png"
|
||||
|
||||
try:
|
||||
await page.screenshot(path=filename, full_page=False)
|
||||
logger.info(f"Screenshot saved to {filename}")
|
||||
except Exception as e:
|
||||
logger.error(f"Screenshot failed: {e}")
|
||||
return None, "", is_available
|
||||
|
||||
# Text Extraction (après le screenshot pour ne pas le perdre)
|
||||
page_text = ""
|
||||
if text_length > 0:
|
||||
try:
|
||||
logger.info(f"Extracting text (limit: {text_length} chars)...")
|
||||
# Get text from body avec un timeout court
|
||||
raw_text = await page.inner_text("body", timeout=10000)
|
||||
# Simple truncation
|
||||
page_text = raw_text[:text_length]
|
||||
logger.info(f"Extracted {len(page_text)} characters")
|
||||
|
||||
# Check for "product not found" messages in page text
|
||||
unavailable_patterns = [
|
||||
"produit introuvable",
|
||||
"produit indisponible",
|
||||
"page introuvable",
|
||||
"404",
|
||||
"product not found",
|
||||
"item not found",
|
||||
"page not found",
|
||||
"n'existe plus",
|
||||
"plus disponible",
|
||||
"no longer available",
|
||||
"article introuvable",
|
||||
"cette page n'existe pas",
|
||||
"this page doesn't exist",
|
||||
]
|
||||
|
||||
page_text_lower = page_text.lower()
|
||||
for pattern in unavailable_patterns:
|
||||
if pattern in page_text_lower:
|
||||
logger.warning(f"Product unavailable - found pattern: '{pattern}'")
|
||||
is_available = False
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Text extraction failed (screenshot saved): {e}")
|
||||
# On continue car le screenshot a été pris
|
||||
|
||||
return filename, page_text, is_available
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping {url}: {e}")
|
||||
return None, "", False # Mark as unavailable on error
|
||||
|
||||
finally:
|
||||
# Fermeture propre des ressources dans l'ordre inverse
|
||||
try:
|
||||
if page and not page.is_closed():
|
||||
await page.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error closing page: {e}")
|
||||
|
||||
try:
|
||||
if context:
|
||||
await context.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error closing context: {e}")
|
||||
|
||||
# On ne ferme le navigateur que s'il est local (non partagé)
|
||||
if local_browser:
|
||||
try:
|
||||
if local_browser.is_connected():
|
||||
await local_browser.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error closing browser: {e}")
|
||||
|
||||
|
||||
# Si on a créé un manager playwright local, on l'arrête
|
||||
if playwright_manager:
|
||||
try:
|
||||
await playwright_manager.__aexit__(None, None, None)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error stopping playwright manager: {e}")
|
||||
+120
-459
@@ -1,483 +1,144 @@
|
||||
"""
|
||||
Search Service - Orchestrateur de recherche de produits
|
||||
Combine recherche directe sur sites + Light Scraper + Browserless fallback
|
||||
New Search Service
|
||||
Orchestrates searches across multiple e-commerce sites using BrowserlessService.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
import re
|
||||
from urllib.parse import quote_plus, urljoin
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from sqlalchemy.orm import Session
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.models import SearchSite
|
||||
from app.schemas import SearchProgress, SearchResultItem
|
||||
from app.services import direct_search_service, light_scraper_service
|
||||
from app.services.ai_service import AIService
|
||||
from app.services.scraper_service import ScraperService
|
||||
from app.services.settings_service import SettingsService
|
||||
from app.core.search_config import SITE_CONFIGS
|
||||
from app.services.browserless_service import browserless_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration par défaut
|
||||
DEFAULT_MAX_RESULTS = 20
|
||||
DEFAULT_PARALLEL_LIMIT = 5
|
||||
DEFAULT_TIMEOUT = 30
|
||||
class SearchResult:
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
snippet: str,
|
||||
source: str,
|
||||
price: float | None = None,
|
||||
currency: str = "EUR",
|
||||
in_stock: bool | None = None,
|
||||
):
|
||||
self.url = url
|
||||
self.title = title
|
||||
self.snippet = snippet
|
||||
self.source = source
|
||||
self.price = price
|
||||
self.currency = currency
|
||||
self.in_stock = in_stock
|
||||
|
||||
|
||||
async def search_products(
|
||||
query: str,
|
||||
db: Session,
|
||||
site_ids: list[int] | None = None,
|
||||
max_results: int | None = None,
|
||||
) -> AsyncGenerator[SearchProgress, None]:
|
||||
"""
|
||||
Recherche des produits sur les sites configurés.
|
||||
|
||||
Flux:
|
||||
1. Recherche directe sur les pages de recherche de chaque site
|
||||
2. Light scraper tente extraction HTTP rapide
|
||||
3. Browserless fallback pour les sites JS
|
||||
|
||||
Args:
|
||||
query: Terme de recherche
|
||||
db: Session de base de données
|
||||
site_ids: IDs des sites à utiliser (None = tous les actifs)
|
||||
max_results: Nombre max de résultats
|
||||
|
||||
Yields:
|
||||
SearchProgress avec les résultats au fur et à mesure
|
||||
"""
|
||||
# Récupérer la configuration
|
||||
if max_results is None:
|
||||
max_results = int(SettingsService.get_setting_value(db, "search_max_results", str(DEFAULT_MAX_RESULTS)))
|
||||
|
||||
parallel_limit = int(SettingsService.get_setting_value(db, "search_parallel_limit", str(DEFAULT_PARALLEL_LIMIT)))
|
||||
|
||||
# Récupérer les sites
|
||||
sites = _get_sites(db, site_ids)
|
||||
if not sites:
|
||||
yield SearchProgress(
|
||||
status="error",
|
||||
total=0,
|
||||
completed=0,
|
||||
message="Aucun site de recherche configuré",
|
||||
results=[],
|
||||
)
|
||||
return
|
||||
|
||||
# Préparer les données des sites pour la recherche directe
|
||||
sites_data = [
|
||||
{
|
||||
"domain": site.domain,
|
||||
"name": site.name,
|
||||
"search_url": site.search_url,
|
||||
"product_link_selector": site.product_link_selector,
|
||||
"requires_js": site.requires_js,
|
||||
"debug_enabled": site.debug_enabled,
|
||||
def to_dict(self):
|
||||
return {
|
||||
"url": self.url,
|
||||
"title": self.title,
|
||||
"snippet": self.snippet,
|
||||
"source": self.source,
|
||||
"price": self.price,
|
||||
"currency": self.currency,
|
||||
"in_stock": self.in_stock,
|
||||
}
|
||||
for site in sites
|
||||
]
|
||||
site_map = {site.domain: site for site in sites}
|
||||
|
||||
# Phase 1: Recherche directe sur les sites
|
||||
yield SearchProgress(
|
||||
status="searching",
|
||||
total=0,
|
||||
completed=0,
|
||||
message=f"Recherche sur {len(sites)} sites...",
|
||||
results=[],
|
||||
)
|
||||
class NewSearchService:
|
||||
@staticmethod
|
||||
async def search_site(site_key: str, query: str) -> list[SearchResult]:
|
||||
"""Search a single site"""
|
||||
config = SITE_CONFIGS.get(site_key)
|
||||
if not config:
|
||||
logger.error(f"Unknown site: {site_key}")
|
||||
return []
|
||||
|
||||
browser = None
|
||||
playwright = None
|
||||
|
||||
try:
|
||||
# Initialiser Playwright et Browserless pour TOUTE la session de recherche
|
||||
logger.info(f"Initialisation session Browserless globale pour '{query}'")
|
||||
playwright = await async_playwright().start()
|
||||
browser = await playwright.chromium.connect_over_cdp(
|
||||
os.getenv("BROWSERLESS_URL", "ws://browserless:3000"),
|
||||
timeout=60000
|
||||
)
|
||||
search_url = config["search_url"].format(query=quote_plus(query))
|
||||
logger.info(f"Searching {config['name']} at {search_url}")
|
||||
|
||||
try:
|
||||
search_results = await direct_search_service.search(
|
||||
query=query,
|
||||
sites=sites_data,
|
||||
max_results=max_results,
|
||||
browser=browser, # Utiliser le navigateur partagé
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur critique lors de la recherche directe: {e}")
|
||||
yield SearchProgress(
|
||||
status="error",
|
||||
total=0,
|
||||
completed=0,
|
||||
message=f"Erreur de recherche: {str(e)}",
|
||||
results=[],
|
||||
)
|
||||
return
|
||||
|
||||
if not search_results:
|
||||
yield SearchProgress(
|
||||
status="completed",
|
||||
total=0,
|
||||
completed=0,
|
||||
message="Aucun résultat trouvé sur les sites configurés",
|
||||
results=[],
|
||||
)
|
||||
return
|
||||
|
||||
total = len(search_results)
|
||||
logger.info(f"Recherche directe: {total} URLs trouvées pour '{query}'")
|
||||
|
||||
# Phase 2: Scraping des URLs
|
||||
results: list[SearchResultItem] = []
|
||||
completed = 0
|
||||
|
||||
# Créer un sémaphore pour limiter la concurrence globale
|
||||
semaphore = asyncio.Semaphore(parallel_limit)
|
||||
# Sémaphore spécifique pour Amazon (1 seul thread à la fois pour éviter le blocage)
|
||||
amazon_semaphore = asyncio.Semaphore(1)
|
||||
|
||||
async def process_url(search_result: direct_search_service.SearchResult) -> SearchResultItem | None:
|
||||
async with semaphore:
|
||||
url = search_result.url
|
||||
domain = search_result.source
|
||||
site = site_map.get(domain)
|
||||
|
||||
if not site:
|
||||
# Trouver le site par correspondance partielle
|
||||
for d, s in site_map.items():
|
||||
if d in domain or domain in d:
|
||||
site = s
|
||||
break
|
||||
|
||||
site_name = site.name if site else domain
|
||||
requires_js = site.requires_js if site else False
|
||||
|
||||
# Essayer le scraping léger d'abord (sauf si le site requiert JS)
|
||||
if not requires_js:
|
||||
light_result = await light_scraper_service.scrape_url(url)
|
||||
|
||||
if light_result.success and light_result.price is not None:
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=light_result.title or search_result.title,
|
||||
price=light_result.price,
|
||||
currency=light_result.currency,
|
||||
in_stock=light_result.in_stock,
|
||||
image_url=light_result.image_url,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
confidence=0.7, # Confiance moyenne pour le scraping léger
|
||||
)
|
||||
|
||||
# Fallback: Browserless + IA
|
||||
try:
|
||||
# Passer le navigateur partagé
|
||||
if "amazon" in domain.lower():
|
||||
async with amazon_semaphore:
|
||||
# Petit délai supplémentaire entre les requêtes Amazon
|
||||
await asyncio.sleep(2.0)
|
||||
result = await _scrape_with_browserless(url, site_name, domain, search_result.title, browser)
|
||||
else:
|
||||
result = await _scrape_with_browserless(url, site_name, domain, search_result.title, browser)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur scraping {url}: {e}")
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=search_result.title,
|
||||
price=None,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# Traiter les URLs en parallèle avec mises à jour progressives
|
||||
tasks = [asyncio.create_task(process_url(r)) for r in search_results]
|
||||
|
||||
for coro in asyncio.as_completed(tasks):
|
||||
try:
|
||||
result = await coro
|
||||
completed += 1
|
||||
|
||||
if result:
|
||||
# Ajouter tous les résultats, même sans prix (le frontend affichera "Prix non disponible")
|
||||
results.append(result)
|
||||
|
||||
yield SearchProgress(
|
||||
status="scraping",
|
||||
total=total,
|
||||
completed=completed,
|
||||
current_site=result.site_name if result else None,
|
||||
results=results.copy(),
|
||||
message=f"Extraction {completed}/{total}...",
|
||||
)
|
||||
except Exception as e:
|
||||
completed += 1
|
||||
logger.error(f"Erreur lors du traitement: {e}")
|
||||
|
||||
# Trier les résultats par prix
|
||||
results.sort(key=lambda x: x.price if x.price else float("inf"))
|
||||
|
||||
yield SearchProgress(
|
||||
status="completed",
|
||||
total=total,
|
||||
completed=completed,
|
||||
results=results,
|
||||
message=f"{len(results)} produits trouvés avec prix",
|
||||
)
|
||||
# Use proxy if required by config
|
||||
use_proxy = config.get("requires_proxy", False)
|
||||
|
||||
finally:
|
||||
# Nettoyage global des ressources
|
||||
if browser:
|
||||
try:
|
||||
if browser.is_connected():
|
||||
await browser.close()
|
||||
logger.info("Navigateur global fermé")
|
||||
except Exception as e:
|
||||
logger.error(f"Erreur fermeture navigateur: {e}")
|
||||
html_content, _ = await browserless_service.get_page_content(
|
||||
search_url,
|
||||
use_proxy=use_proxy,
|
||||
wait_selector=config.get("wait_selector")
|
||||
)
|
||||
|
||||
if not html_content:
|
||||
logger.warning(f"No content returned for {site_key}")
|
||||
return []
|
||||
|
||||
return NewSearchService._parse_results(html_content, site_key, search_url)
|
||||
|
||||
@staticmethod
|
||||
def _parse_results(html: str, site_key: str, base_url: str) -> list[SearchResult]:
|
||||
"""Parse HTML content to extract search results"""
|
||||
config = SITE_CONFIGS[site_key]
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
# Select product links
|
||||
links = soup.select(config["product_selector"])
|
||||
|
||||
# Petit délai pour laisser le temps aux connexions de se fermer proprement
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if playwright:
|
||||
try:
|
||||
await playwright.stop()
|
||||
except Exception as e:
|
||||
# Ignorer les erreurs courantes lors de l'arrêt si déjà fermé
|
||||
if "Event loop is closed" not in str(e):
|
||||
logger.error(f"Erreur arrêt Playwright: {e}")
|
||||
# Deduplicate links
|
||||
seen_urls = set()
|
||||
|
||||
for link in links:
|
||||
if len(results) >= 5: # Limit to 5 results per site
|
||||
break
|
||||
|
||||
href = link.get("href")
|
||||
if not href:
|
||||
continue
|
||||
|
||||
async def _scrape_with_browserless(
|
||||
url: str,
|
||||
site_name: str,
|
||||
domain: str,
|
||||
fallback_title: str,
|
||||
browser=None, # Navigateur partagé
|
||||
) -> SearchResultItem:
|
||||
"""Scrape une URL avec Browserless + extraction IA"""
|
||||
try:
|
||||
# Scraper avec Browserless (utiliser le navigateur partagé)
|
||||
screenshot_path, page_text, is_available = await ScraperService.scrape_item(
|
||||
url=url,
|
||||
item_id=None,
|
||||
smart_scroll=True,
|
||||
scroll_pixels=350,
|
||||
text_length=3000,
|
||||
timeout=30000,
|
||||
browser=browser,
|
||||
)
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
# Basic cleanup
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
if not screenshot_path:
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=fallback_title,
|
||||
price=None,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
error="Screenshot failed",
|
||||
)
|
||||
# Extract title
|
||||
title = link.get_text(strip=True)
|
||||
if not title:
|
||||
# Try finding title in children or attributes
|
||||
img = link.find("img")
|
||||
if img and img.get("alt"):
|
||||
title = img.get("alt")
|
||||
elif link.get("title"):
|
||||
title = link.get("title")
|
||||
|
||||
if not title or len(title) < 3:
|
||||
continue
|
||||
|
||||
# Extraction IA
|
||||
ai_result = await AIService.analyze_image(
|
||||
image_path=screenshot_path,
|
||||
page_text=page_text,
|
||||
)
|
||||
# Create result
|
||||
results.append(SearchResult(
|
||||
url=full_url,
|
||||
title=title,
|
||||
snippet=f"Product from {config['name']}",
|
||||
source=config["name"]
|
||||
))
|
||||
|
||||
# Construire l'URL du screenshot pour le frontend
|
||||
screenshot_url = None
|
||||
if screenshot_path:
|
||||
screenshot_filename = os.path.basename(screenshot_path)
|
||||
screenshot_url = f"/screenshots/{screenshot_filename}"
|
||||
logger.info(f"Found {len(results)} results for {site_key}")
|
||||
return results
|
||||
|
||||
if ai_result:
|
||||
extraction, metadata = ai_result
|
||||
if extraction and extraction.price is not None:
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=fallback_title,
|
||||
price=extraction.price,
|
||||
currency=extraction.currency or "EUR",
|
||||
in_stock=extraction.in_stock,
|
||||
image_url=screenshot_url,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
confidence=extraction.price_confidence or 0.5,
|
||||
)
|
||||
@staticmethod
|
||||
async def search_all(query: str) -> list[SearchResult]:
|
||||
"""Search all configured sites in parallel"""
|
||||
tasks = []
|
||||
for site_key in SITE_CONFIGS.keys():
|
||||
tasks.append(NewSearchService.search_site(site_key, query))
|
||||
|
||||
results_list = await asyncio.gather(*tasks)
|
||||
|
||||
# Flatten list
|
||||
all_results = []
|
||||
for site_results in results_list:
|
||||
all_results.extend(site_results)
|
||||
|
||||
return all_results
|
||||
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=fallback_title,
|
||||
price=None,
|
||||
image_url=screenshot_url,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
error="AI extraction failed",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Browserless scraping error for {url}: {e}")
|
||||
return SearchResultItem(
|
||||
url=url,
|
||||
title=fallback_title,
|
||||
price=None,
|
||||
site_name=site_name,
|
||||
site_domain=domain,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
def _get_sites(db: Session, site_ids: list[int] | None = None) -> list[SearchSite]:
|
||||
"""Récupère les sites de recherche actifs"""
|
||||
query = db.query(SearchSite).filter(SearchSite.is_active == True)
|
||||
|
||||
if site_ids:
|
||||
query = query.filter(SearchSite.id.in_(site_ids))
|
||||
|
||||
return query.order_by(SearchSite.priority).all()
|
||||
|
||||
|
||||
def get_all_sites(db: Session) -> list[SearchSite]:
|
||||
"""Récupère tous les sites de recherche"""
|
||||
return db.query(SearchSite).order_by(SearchSite.priority).all()
|
||||
|
||||
|
||||
def get_site_by_id(db: Session, site_id: int) -> SearchSite | None:
|
||||
"""Récupère un site par son ID"""
|
||||
return db.query(SearchSite).filter(SearchSite.id == site_id).first()
|
||||
|
||||
|
||||
def create_site(db: Session, site_data: dict[str, Any]) -> SearchSite:
|
||||
"""Crée un nouveau site de recherche"""
|
||||
site = SearchSite(**site_data)
|
||||
db.add(site)
|
||||
db.commit()
|
||||
db.refresh(site)
|
||||
return site
|
||||
|
||||
|
||||
def update_site(db: Session, site_id: int, site_data: dict[str, Any]) -> SearchSite | None:
|
||||
"""Met à jour un site de recherche"""
|
||||
site = get_site_by_id(db, site_id)
|
||||
if not site:
|
||||
return None
|
||||
|
||||
for key, value in site_data.items():
|
||||
if value is not None:
|
||||
setattr(site, key, value)
|
||||
|
||||
db.commit()
|
||||
db.refresh(site)
|
||||
return site
|
||||
|
||||
|
||||
def delete_site(db: Session, site_id: int) -> bool:
|
||||
"""Supprime un site de recherche"""
|
||||
site = get_site_by_id(db, site_id)
|
||||
if not site:
|
||||
return False
|
||||
|
||||
db.delete(site)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def seed_default_sites(db: Session) -> int:
|
||||
"""
|
||||
Initialise la base de données avec les sites configurés en dur.
|
||||
Ne crée que les sites qui n'existent pas encore (basé sur le domaine).
|
||||
|
||||
Returns:
|
||||
Nombre de sites créés
|
||||
"""
|
||||
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
|
||||
|
||||
created_count = 0
|
||||
|
||||
# Récupérer les domaines existants
|
||||
existing_domains = {site.domain.lower().replace("www.", "")
|
||||
for site in db.query(SearchSite).all()}
|
||||
|
||||
for domain, config in DEFAULT_SITE_CONFIGS.items():
|
||||
# Normaliser le domaine pour comparaison
|
||||
clean_domain = domain.lower().replace("www.", "")
|
||||
|
||||
# Ne pas créer si déjà existant
|
||||
if clean_domain in existing_domains:
|
||||
logger.debug(f"Site {domain} déjà existant, ignoré")
|
||||
continue
|
||||
|
||||
# Créer le site
|
||||
site_data = {
|
||||
"name": config.get("name", domain),
|
||||
"domain": domain,
|
||||
"search_url": config.get("search_url"),
|
||||
"product_link_selector": config.get("product_selector"),
|
||||
"category": config.get("category"),
|
||||
"requires_js": config.get("requires_js", True),
|
||||
"priority": config.get("priority", 99),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
try:
|
||||
site = SearchSite(**site_data)
|
||||
db.add(site)
|
||||
db.commit()
|
||||
created_count += 1
|
||||
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Erreur création site {domain}: {e}")
|
||||
|
||||
return created_count
|
||||
|
||||
|
||||
def reset_sites_to_defaults(db: Session) -> int:
|
||||
"""
|
||||
Réinitialise tous les sites avec les valeurs par défaut.
|
||||
Supprime tous les sites existants et recrée à partir de la config.
|
||||
|
||||
Returns:
|
||||
Nombre de sites créés
|
||||
"""
|
||||
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
|
||||
|
||||
# Supprimer tous les sites existants
|
||||
db.query(SearchSite).delete()
|
||||
db.commit()
|
||||
logger.info("Tous les sites supprimés")
|
||||
|
||||
# Recréer avec les valeurs par défaut
|
||||
created_count = 0
|
||||
|
||||
for domain, config in DEFAULT_SITE_CONFIGS.items():
|
||||
site_data = {
|
||||
"name": config.get("name", domain),
|
||||
"domain": domain,
|
||||
"search_url": config.get("search_url"),
|
||||
"product_link_selector": config.get("product_selector"),
|
||||
"category": config.get("category"),
|
||||
"requires_js": config.get("requires_js", True),
|
||||
"priority": config.get("priority", 99),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
try:
|
||||
site = SearchSite(**site_data)
|
||||
db.add(site)
|
||||
db.commit()
|
||||
created_count += 1
|
||||
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Erreur création site {domain}: {e}")
|
||||
|
||||
return created_count
|
||||
# Global instance
|
||||
new_search_service = NewSearchService()
|
||||
@@ -0,0 +1,53 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add project root to path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".")))
|
||||
|
||||
from app.services.search_service import new_search_service
|
||||
from app.services.browserless_service import browserless_service
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
handlers=[logging.StreamHandler()]
|
||||
)
|
||||
|
||||
async def verify_search():
|
||||
query = "iphone 15"
|
||||
print(f"--- Starting Verification Search for '{query}' ---")
|
||||
|
||||
# 1. Test Browserless Connection
|
||||
print("\n[1] Testing Browserless Connection...")
|
||||
try:
|
||||
await browserless_service.start()
|
||||
print("✅ Browserless connected successfully")
|
||||
except Exception as e:
|
||||
print(f"❌ Browserless connection failed: {e}")
|
||||
return
|
||||
|
||||
# 2. Test Specific Sites
|
||||
sites_to_test = ["gifi.fr", "amazon.fr", "e.leclerc"]
|
||||
|
||||
for site in sites_to_test:
|
||||
print(f"\n[2] Testing Search on {site}...")
|
||||
try:
|
||||
results = await new_search_service.search_site(site, query)
|
||||
if results:
|
||||
print(f"✅ Found {len(results)} results for {site}")
|
||||
for r in results[:2]:
|
||||
print(f" - {r.title} ({r.url})")
|
||||
else:
|
||||
print(f"⚠️ No results found for {site}")
|
||||
except Exception as e:
|
||||
print(f"❌ Error searching {site}: {e}")
|
||||
|
||||
# 3. Cleanup
|
||||
await browserless_service.stop()
|
||||
print("\n--- Verification Complete ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(verify_search())
|
||||
Reference in new issue
Block a user