mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-12 18:00:00 +02:00
459 lines
20 KiB
Python
459 lines
20 KiB
Python
import asyncio
|
|
import logging
|
|
import os
|
|
import re
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
|
from playwright.async_api import async_playwright
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
|
|
|
|
# Nombre de tentatives pour le scraping
|
|
MAX_RETRIES = 2
|
|
RETRY_DELAY = 3 # secondes
|
|
|
|
|
|
def simplify_url(url: str) -> str:
|
|
"""
|
|
Simplifie les URLs en supprimant les paramètres de tracking.
|
|
Particulièrement utile pour Amazon qui a des URLs très longues.
|
|
|
|
Exemples:
|
|
- Amazon: https://www.amazon.fr/dp/B0CFYHHPPV?ref=... -> https://www.amazon.fr/dp/B0CFYHHPPV
|
|
- Autres: garde l'URL originale si pas de simplification possible
|
|
"""
|
|
try:
|
|
parsed = urlparse(url)
|
|
hostname = parsed.hostname or ""
|
|
|
|
# Simplification pour Amazon
|
|
if "amazon" in hostname:
|
|
# Extraire l'ASIN (identifiant produit Amazon)
|
|
# Format: /dp/ASIN ou /gp/product/ASIN
|
|
asin_match = re.search(r"/(?:dp|gp/product)/([A-Z0-9]{10})", url)
|
|
if asin_match:
|
|
asin = asin_match.group(1)
|
|
# Reconstruire une URL simple
|
|
simplified = f"{parsed.scheme}://{parsed.netloc}/dp/{asin}"
|
|
logger.info(f"URL Amazon simplifiée: {url[:80]}... -> {simplified}")
|
|
return simplified
|
|
|
|
# Pour les autres sites, supprimer les paramètres de tracking courants
|
|
# mais garder les paramètres essentiels
|
|
tracking_params = [
|
|
"utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term",
|
|
"ref", "ref_", "tag", "linkCode", "linkId", "camp", "creative",
|
|
"crid", "dib", "dib_tag", "qid", "sprefix", "sr", "keywords",
|
|
"fbclid", "gclid", "msclkid"
|
|
]
|
|
|
|
if parsed.query:
|
|
# Garder seulement les paramètres non-tracking
|
|
from urllib.parse import parse_qs, urlencode
|
|
params = parse_qs(parsed.query)
|
|
filtered_params = {
|
|
k: v[0] for k, v in params.items()
|
|
if k.lower() not in tracking_params
|
|
}
|
|
new_query = urlencode(filtered_params) if filtered_params else ""
|
|
simplified = urlunparse((
|
|
parsed.scheme,
|
|
parsed.netloc,
|
|
parsed.path,
|
|
parsed.params,
|
|
new_query,
|
|
"" # fragment
|
|
))
|
|
if simplified != url:
|
|
logger.info(f"URL simplifiée: {url[:60]}... -> {simplified[:60]}...")
|
|
return simplified
|
|
|
|
return url
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Erreur lors de la simplification de l'URL: {e}")
|
|
return url
|
|
|
|
|
|
class ScraperService:
|
|
@staticmethod
|
|
async def scrape_item( # noqa: PLR0913, PLR0912, PLR0915
|
|
url: str,
|
|
selector: str | None = None,
|
|
item_id: int | None = None,
|
|
smart_scroll: bool = False,
|
|
scroll_pixels: int = 350,
|
|
text_length: int = 0,
|
|
timeout: int = 90000,
|
|
browser=None, # Instance de navigateur partagée optionnelle
|
|
) -> tuple[str | None, str, bool]:
|
|
"""
|
|
Scrapes the given URL using Browserless and Playwright.
|
|
Returns a tuple: (screenshot_path, page_text, is_available)
|
|
|
|
Args:
|
|
url: Target URL to scrape
|
|
selector: Optional CSS selector to focus on
|
|
item_id: Optional item ID for screenshot naming
|
|
smart_scroll: Enable scrolling to load lazy content
|
|
scroll_pixels: Number of pixels to scroll (must be positive)
|
|
text_length: Number of characters to extract (0 = disabled)
|
|
timeout: Page load timeout in milliseconds
|
|
browser: Optional shared Playwright browser instance
|
|
"""
|
|
# Simplifier l'URL avant le scraping
|
|
original_url = url
|
|
url = simplify_url(url)
|
|
|
|
# Input validation
|
|
if scroll_pixels <= 0:
|
|
logger.warning(f"Invalid scroll_pixels value: {scroll_pixels}, using default 350")
|
|
scroll_pixels = 350
|
|
|
|
if timeout <= 0:
|
|
logger.warning(f"Invalid timeout value: {timeout}, using default 90000")
|
|
timeout = 90000
|
|
|
|
# Retries avec backoff
|
|
last_error = None
|
|
for attempt in range(MAX_RETRIES + 1):
|
|
if attempt > 0:
|
|
logger.info(f"Tentative {attempt + 1}/{MAX_RETRIES + 1} pour {url}")
|
|
await asyncio.sleep(RETRY_DELAY * attempt)
|
|
|
|
result = await ScraperService._do_scrape(
|
|
url=url,
|
|
selector=selector,
|
|
item_id=item_id,
|
|
smart_scroll=smart_scroll,
|
|
scroll_pixels=scroll_pixels,
|
|
text_length=text_length,
|
|
timeout=timeout,
|
|
browser=browser,
|
|
)
|
|
|
|
if result[0] is not None: # Screenshot réussi
|
|
return result
|
|
|
|
last_error = "Scraping failed"
|
|
logger.warning(f"Tentative {attempt + 1} échouée pour {url}")
|
|
|
|
logger.error(f"Toutes les tentatives ont échoué pour {url}")
|
|
return None, "", False # Product unavailable if all retries failed
|
|
|
|
@staticmethod
|
|
async def _do_scrape( # noqa: PLR0913, PLR0912, PLR0915
|
|
url: str,
|
|
selector: str | None = None,
|
|
item_id: int | None = None,
|
|
smart_scroll: bool = False,
|
|
scroll_pixels: int = 350,
|
|
text_length: int = 0,
|
|
timeout: int = 90000,
|
|
browser=None,
|
|
) -> tuple[str | None, str, bool]:
|
|
"""Exécute le scraping réel (appelé par scrape_item avec retries)."""
|
|
|
|
# Si un navigateur est fourni, on l'utilise directement sans async_playwright context manager
|
|
# sinon on crée tout de zéro
|
|
playwright_manager = None
|
|
local_browser = None
|
|
|
|
try:
|
|
if browser:
|
|
# Utiliser le navigateur partagé
|
|
current_browser = browser
|
|
else:
|
|
# Créer un nouveau navigateur
|
|
playwright_manager = async_playwright()
|
|
p = await playwright_manager.start()
|
|
logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
|
|
current_browser = await p.chromium.connect_over_cdp(
|
|
BROWSERLESS_URL,
|
|
timeout=60000,
|
|
)
|
|
local_browser = current_browser
|
|
|
|
context = None
|
|
page = None
|
|
is_available = True # Par défaut, le produit est disponible
|
|
|
|
try:
|
|
context = await current_browser.new_context(
|
|
viewport={"width": 1920, "height": 1080},
|
|
user_agent=(
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/120.0.0.0 Safari/537.36"
|
|
),
|
|
locale="fr-FR",
|
|
timezone_id="Europe/Paris",
|
|
# Options supplémentaires pour éviter la détection
|
|
extra_http_headers={
|
|
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
}
|
|
)
|
|
|
|
# Stealth mode / Ad blocking attempts
|
|
await context.route("**/*", lambda route: route.continue_())
|
|
|
|
page = await context.new_page()
|
|
|
|
logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
|
|
response = None
|
|
try:
|
|
# First wait for domcontentloaded - this is the minimum we need
|
|
response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
|
|
|
# Check HTTP status code
|
|
if response:
|
|
status = response.status
|
|
logger.info(f"HTTP Status: {status}")
|
|
|
|
# Detect unavailable products by HTTP status
|
|
if status in [404, 410, 451, 503]:
|
|
logger.warning(f"Product unavailable - HTTP {status}")
|
|
is_available = False
|
|
elif status >= 400:
|
|
logger.warning(f"HTTP error {status}, marking as potentially unavailable")
|
|
is_available = False
|
|
|
|
logger.info(f"Page loaded (domcontentloaded): {url}")
|
|
|
|
# Then try to wait for networkidle, but don't fail if it times out
|
|
# This helps with heavy pages that never fully settle
|
|
try:
|
|
await page.wait_for_load_state("networkidle", timeout=5000)
|
|
logger.info("Network idle reached")
|
|
except PlaywrightTimeoutError:
|
|
logger.info("Network idle timed out (non-critical), proceeding...")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error navigating to {url}: {e}")
|
|
# If navigation fails completely, mark as unavailable
|
|
is_available = False
|
|
# Try to take screenshot anyway if page partially loaded
|
|
pass
|
|
|
|
# Wait a bit for dynamic content if needed
|
|
await page.wait_for_timeout(2000)
|
|
|
|
# Try to close common popups and cookie banners
|
|
logger.info("Attempting to close popups and cookie banners...")
|
|
popup_selectors = [
|
|
# === SITES SPÉCIFIQUES FRANÇAIS ===
|
|
# Amazon France
|
|
"#sp-cc-accept",
|
|
"#sp-cc-rejectall-link",
|
|
"input[data-action-type='DISMISS']",
|
|
"#a-popover-content-1 button",
|
|
"[data-action='a-popover-close']",
|
|
# E.Leclerc, Darty, Fnac
|
|
"#onetrust-accept-btn-handler",
|
|
".onetrust-accept-btn-handler",
|
|
# Auchan
|
|
"#popin_tc_privacy_button_2",
|
|
".popin_tc_privacy_button",
|
|
"#didomi-notice-agree-button",
|
|
# Carrefour
|
|
"[data-testid='accept-cookies-button']",
|
|
# Cdiscount
|
|
"#footer_tc_privacy_button_2",
|
|
".privacy_prompt_accept",
|
|
# Boulanger
|
|
".bcom-consent-accept-all",
|
|
"#cookieBanner-accept",
|
|
# Gifi, Centrakor, La Foir'Fouille (Magento)
|
|
".action-primary.action-accept",
|
|
"#btn-cookie-allow",
|
|
# PrestaShop (Stokomani, B&M, etc.)
|
|
".btn-primary[data-dismiss='modal']",
|
|
"#gdpr_consent_agree",
|
|
|
|
# === CMP (Consent Management Platforms) ===
|
|
".didomi-continue-without-agreeing",
|
|
"#tarteaucitronPersonalize2",
|
|
".tarteaucitronAllow",
|
|
"#tarteaucitronAllDenied2",
|
|
".cc-btn.cc-allow",
|
|
".cky-btn-accept",
|
|
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
|
|
"#CybotCookiebotDialogBodyButtonAccept",
|
|
".qc-cmp2-summary-buttons button:first-child",
|
|
"#axeptio_btn_acceptAll",
|
|
".axeptio_acceptAll",
|
|
|
|
# === TEXTE FRANÇAIS ===
|
|
"button:has-text('Tout accepter')",
|
|
"button:has-text('Accepter tout')",
|
|
"button:has-text('Accepter et fermer')",
|
|
"button:has-text('Accepter les cookies')",
|
|
"button:has-text('Accepter')",
|
|
"button:has-text('J\\'accepte')",
|
|
"button:has-text('OK')",
|
|
"button:has-text('Continuer')",
|
|
"a:has-text('Tout accepter')",
|
|
"a:has-text('Accepter')",
|
|
|
|
# === POPUPS GÉNÉRAUX ===
|
|
"button[aria-label='Close']",
|
|
"button[aria-label='close']",
|
|
"button[aria-label='Fermer']",
|
|
".close-button",
|
|
".modal-close",
|
|
"svg[data-name='Close']",
|
|
"[class*='popup'] button",
|
|
"[class*='modal'] button",
|
|
"button:has-text('No, thanks')",
|
|
"button:has-text('No thanks')",
|
|
"button:has-text('Accept')",
|
|
"a:has-text('No, thanks')",
|
|
"div[role='dialog'] button[aria-label='Close']",
|
|
]
|
|
|
|
for popup_selector in popup_selectors:
|
|
try:
|
|
if await page.locator(popup_selector).count() > 0:
|
|
logger.info(f"Found popup close button: {popup_selector}")
|
|
# Try to click it. If it fails, catch and continue
|
|
await page.locator(popup_selector).first.click(timeout=2000)
|
|
await page.wait_for_timeout(1000) # Wait for animation
|
|
except Exception as e:
|
|
logger.debug(f"Could not close popup with selector {popup_selector}: {e}")
|
|
|
|
# Also try pressing Escape
|
|
try:
|
|
await page.keyboard.press("Escape")
|
|
except Exception as e:
|
|
logger.debug(f"Could not press Escape key: {e}")
|
|
|
|
if selector:
|
|
try:
|
|
logger.info(f"Waiting for selector: {selector}")
|
|
await page.wait_for_selector(selector, timeout=5000)
|
|
# Scroll to element
|
|
element = page.locator(selector).first
|
|
await element.scroll_into_view_if_needed()
|
|
logger.info(f"Scrolled to selector: {selector}")
|
|
except Exception as e:
|
|
logger.warning(f"Selector {selector} not found or timed out: {e}")
|
|
else:
|
|
# Auto-detect price if no selector
|
|
logger.info("No selector provided. Attempting to find price element...")
|
|
try:
|
|
# Look for common price patterns
|
|
price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/")
|
|
if await price_locator.count() > 0:
|
|
# Pick the first one that looks visible and reasonable size
|
|
# This is heuristic
|
|
await price_locator.first.scroll_into_view_if_needed()
|
|
logger.info("Scrolled to potential price element")
|
|
except Exception as e:
|
|
logger.warning(f"Auto-price detection failed: {e}")
|
|
|
|
# Smart Scroll
|
|
if smart_scroll:
|
|
logger.info(f"Performing smart scroll ({scroll_pixels}px)...")
|
|
try:
|
|
await page.evaluate(f"window.scrollBy(0, {scroll_pixels})")
|
|
await page.wait_for_timeout(1000)
|
|
except Exception as e:
|
|
logger.warning(f"Smart scroll failed: {e}")
|
|
|
|
# IMPORTANT: Prendre le screenshot AVANT l'extraction de texte
|
|
# pour éviter de perdre le screenshot si le navigateur se ferme
|
|
screenshot_dir = "screenshots"
|
|
os.makedirs(screenshot_dir, exist_ok=True)
|
|
if item_id:
|
|
filename = f"{screenshot_dir}/item_{item_id}.png"
|
|
else:
|
|
url_part = url.split("//")[-1].replace("/", "_")[:50] # Limiter la longueur
|
|
timestamp = int(asyncio.get_event_loop().time())
|
|
filename = f"{screenshot_dir}/{url_part}_{timestamp}.png"
|
|
|
|
try:
|
|
await page.screenshot(path=filename, full_page=False)
|
|
logger.info(f"Screenshot saved to {filename}")
|
|
except Exception as e:
|
|
logger.error(f"Screenshot failed: {e}")
|
|
return None, "", is_available
|
|
|
|
# Text Extraction (après le screenshot pour ne pas le perdre)
|
|
page_text = ""
|
|
if text_length > 0:
|
|
try:
|
|
logger.info(f"Extracting text (limit: {text_length} chars)...")
|
|
# Get text from body avec un timeout court
|
|
raw_text = await page.inner_text("body", timeout=10000)
|
|
# Simple truncation
|
|
page_text = raw_text[:text_length]
|
|
logger.info(f"Extracted {len(page_text)} characters")
|
|
|
|
# Check for "product not found" messages in page text
|
|
unavailable_patterns = [
|
|
"produit introuvable",
|
|
"produit indisponible",
|
|
"page introuvable",
|
|
"404",
|
|
"product not found",
|
|
"item not found",
|
|
"page not found",
|
|
"n'existe plus",
|
|
"plus disponible",
|
|
"no longer available",
|
|
"article introuvable",
|
|
"cette page n'existe pas",
|
|
"this page doesn't exist",
|
|
]
|
|
|
|
page_text_lower = page_text.lower()
|
|
for pattern in unavailable_patterns:
|
|
if pattern in page_text_lower:
|
|
logger.warning(f"Product unavailable - found pattern: '{pattern}'")
|
|
is_available = False
|
|
break
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Text extraction failed (screenshot saved): {e}")
|
|
# On continue car le screenshot a été pris
|
|
|
|
return filename, page_text, is_available
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scraping {url}: {e}")
|
|
return None, "", False # Mark as unavailable on error
|
|
|
|
finally:
|
|
# Fermeture propre des ressources dans l'ordre inverse
|
|
try:
|
|
if page and not page.is_closed():
|
|
await page.close()
|
|
except Exception as e:
|
|
logger.debug(f"Error closing page: {e}")
|
|
|
|
try:
|
|
if context:
|
|
await context.close()
|
|
except Exception as e:
|
|
logger.debug(f"Error closing context: {e}")
|
|
|
|
# On ne ferme le navigateur que s'il est local (non partagé)
|
|
if local_browser:
|
|
try:
|
|
if local_browser.is_connected():
|
|
await local_browser.close()
|
|
except Exception as e:
|
|
logger.debug(f"Error closing browser: {e}")
|
|
|
|
finally:
|
|
# Si on a créé un manager playwright local, on l'arrête
|
|
if playwright_manager:
|
|
try:
|
|
await playwright_manager.__aexit__(None, None, None)
|
|
except Exception as e:
|
|
logger.debug(f"Error stopping playwright manager: {e}")
|