Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-24 07:38:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #129 from R0m1k3/antigravity

Browse Source 
feat: Implement Bonial.fr scraper service using Playwright to extract…
This commit is contained in:
LogiFlow authored and GitHub committed 2025-11-29 23:26:07 +01:00
2 parents f010efc1f8 b7f049d414
commit 92b3f7ed33
1 file changed
+57 -178
Download Patch File Download Diff File Expand all files Collapse all files
app/services/bonial_scraper.py
+57 -178
View File
@@ -9,6 +9,7 @@ import asyncio
import hashlib import hashlib
import logging import logging
import re import re
import os
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any
@@ -19,17 +20,15 @@ from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Bonial selectors based on browser inspection # Bonial CSS selectors (based on browser inspection - updated for new structure)
BONIAL_SELECTORS = { BONIAL_SELECTORS = {
# Card must contain a catalog link AND an image "catalog_card": "div:has(> a[href*='/contentViewer/'])",
"catalog_card": "section:has(a[href*='/catalogue/']), div[class*='brochure-card']", "catalog_title": "a[href*='/contentViewer/'] h2",
"catalog_title": "div[class*='title'], h2, h3, h4, .text-lg", "catalog_image": "a[href*='/contentViewer/'] img",
"catalog_image": "img[src*='content-media.bonial.biz']", "catalog_link": "a[href*='/contentViewer/']",
"catalog_link": "a[href*='/catalogue/']", "cookie_accept": "button:has-text('Accepter')",
"cookie_accept": "button:has-text('OK'), button:has-text('Accepter'), #onetrust-accept-btn-handler",
} }
# Date pattern: matches "lun. 25/11 - mar. 08/12/2025" or similar
DATE_PATTERN = r"(\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)" DATE_PATTERN = r"(\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)"
@@ -46,15 +45,10 @@ def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
Raises: Raises:
ValueError: If date format is invalid ValueError: If date format is invalid
""" """
# Find all dates in the string
matches = re.findall(DATE_PATTERN, date_str) matches = re.findall(DATE_PATTERN, date_str)
if not matches: if not matches:
# Fallback: try to find "Valable ... jours" or similar
# For now, default to today + 7 days if no date found
logger.warning(f"No dates found in '{date_str}', using default duration") logger.warning(f"No dates found in '{date_str}', using default duration")
return datetime.now(), datetime.now().replace(year=datetime.now().year + 1) # Valid for 1 year if unknown? No, better to be safe.
# Actually, let's raise error to filter out bad data, or handle gracefully
raise ValueError(f"Invalid date format: {date_str}") raise ValueError(f"Invalid date format: {date_str}")
today = datetime.now() today = datetime.now()
@@ -62,7 +56,6 @@ def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
parsed_dates = [] parsed_dates = []
for date_match in matches: for date_match in matches:
# Normalize separators
date_match = date_match.replace("-", "/") date_match = date_match.replace("-", "/")
parts = date_match.split("/") parts = date_match.split("/")
@@ -70,16 +63,12 @@ def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
month = int(parts[1]) month = int(parts[1])
year = int(parts[2]) if len(parts) > 2 else current_year year = int(parts[2]) if len(parts) > 2 else current_year
# Handle 2-digit years
if year < 100: if year < 100:
year += 2000 year += 2000
# Handle year rollover (e.g. in Dec, date is Jan) if len(parts) == 2:
if len(parts) == 2: # No year specified if month < today.month - 3:
if month < today.month - 3: # Date is in past months -> likely next year
year += 1 year += 1
elif month > today.month + 3: # Date is far in future -> likely current year (or previous if looking at old catalogs)
pass
try: try:
parsed_dates.append(datetime(year, month, day)) parsed_dates.append(datetime(year, month, day))
@@ -92,28 +81,17 @@ def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
if len(parsed_dates) >= 2: if len(parsed_dates) >= 2:
return min(parsed_dates), max(parsed_dates) return min(parsed_dates), max(parsed_dates)
elif len(parsed_dates) == 1: elif len(parsed_dates) == 1:
# If only one date, assume it's end date if it's in the future
date = parsed_dates[0] date = parsed_dates[0]
if date > today: if date > today:
return today, date return today, date
else: else:
return date, date # Expired or start date? Assume start return date, date
return parsed_dates[0], parsed_dates[0] return parsed_dates[0], parsed_dates[0]
def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str: def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str:
""" """Compute SHA256 hash for duplicate detection."""
Compute SHA256 hash for duplicate detection.
Args:
enseigne_id: ID of the enseigne
titre: Catalog title
date_debut: Start date
Returns:
SHA256 hash string
"""
content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}" content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}"
return hashlib.sha256(content.encode()).hexdigest() return hashlib.sha256(content.encode()).hexdigest()
@@ -128,34 +106,20 @@ async def accept_cookies(page: Page) -> None:
logger.debug("No cookie banner found or already accepted") logger.debug("No cookie banner found or already accepted")
async def scrape_catalog_list( async def scrape_catalog_list(page: Page, enseigne: Enseigne) -> list[dict[str, Any]]:
page: Page, """Scrape the list of catalogs for an enseigne from Bonial."""
enseigne: Enseigne
) -> list[dict[str, Any]]:
"""
Scrape the list of catalogs for an ens enseigne from Bonial.
Args:
page: Playwright page object
enseigne: Enseigne object
Returns:
List of catalog data dictionaries
"""
url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}" url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}"
logger.info(f"Scraping catalog list for {enseigne.nom} from {url}") logger.info(f"Scraping catalog list for {enseigne.nom} from {url}")
await page.goto(url, wait_until="networkidle") await page.goto(url, wait_until="networkidle")
await accept_cookies(page) await accept_cookies(page)
# Wait for catalog cards to load
try: try:
await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000) await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000)
except Exception as e: except Exception as e:
logger.error(f"No catalog cards found for {enseigne.nom}: {e}") logger.error(f"No catalog cards found for {enseigne.nom}: {e}")
return [] return []
# Extract all catalog cards
cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"]) cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"])
logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}") logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}")
@@ -163,80 +127,52 @@ async def scrape_catalog_list(
for idx, card in enumerate(cards): for idx, card in enumerate(cards):
try: try:
# Extract title
title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"]) title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"])
titre = await title_el.inner_text() if title_el else "Catalogue" titre = await title_el.inner_text() if title_el else "Catalogue"
# Extract dates - look for any element containing date pattern
dates_text = await card.inner_text() dates_text = await card.inner_text()
# Extract image
image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"]) image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"])
image_url = await image_el.get_attribute("src") if image_el else None image_url = await image_el.get_attribute("src") if image_el else None
if image_url and image_url.startswith("//"): if image_url and image_url.startswith("//"):
image_url = "https:" + image_url image_url = "https:" + image_url
# Extract link - try multiple strategies link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"])
# Strategy 1: The card itself is a link catalogue_url = await link_el.get_attribute("href") if link_el else None
catalogue_url = await card.get_attribute("href")
# Strategy 2: Look for specific catalog link inside
if not catalogue_url:
link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"])
if link_el:
catalogue_url = await link_el.get_attribute("href")
# Strategy 3: Look for ANY link inside
if not catalogue_url:
link_el = await card.query_selector("a")
if link_el:
catalogue_url = await link_el.get_attribute("href")
# Make URL absolute if relative
if catalogue_url and not catalogue_url.startswith("http"): if catalogue_url and not catalogue_url.startswith("http"):
catalogue_url = f"https://www.bonial.fr{catalogue_url}" catalogue_url = f"https://www.bonial.fr{catalogue_url}"
# Validate URL - Must be a real catalog viewer, not an app link or generic page # Validate URL
if catalogue_url and ("adjust.com" in catalogue_url or "/Mobile" in catalogue_url): if catalogue_url and ("adjust.com" in catalogue_url or "/Mobile" in catalogue_url):
logger.warning(f"Skipping app/mobile link for catalog {idx}: {catalogue_url}") logger.debug(f"Skipping app/mobile link: {catalogue_url}")
catalogue_url = None continue
if catalogue_url and "/catalogue/" not in catalogue_url: if catalogue_url and "/contentViewer/" not in catalogue_url:
logger.warning(f"Skipping non-catalog URL for catalog {idx}: {catalogue_url}") logger.debug(f"Skipping non-catalog URL: {catalogue_url}")
catalogue_url = None continue
# Filter out generic banners/newsletters based on title # Filter generic titles
if titre: if titre:
lower_title = titre.lower() lower_title = titre.lower()
ignored_titles = [ ignored_titles = [
"restez informé", "restez informé", "catalogues bazar", "toutes les offres",
"catalogues bazar", "voir les offres", "téléchargez l'application", "newsletter"
"toutes les offres",
"voir les offres",
"téléchargez l'application",
"newsletter"
] ]
if any(ignored in lower_title for ignored in ignored_titles): if any(ignored in lower_title for ignored in ignored_titles):
logger.info(f"Skipping generic/banner card: {titre}") logger.info(f"Skipping generic/banner card: {titre}")
continue continue
# Validate data
has_url = bool(catalogue_url) has_url = bool(catalogue_url)
has_image = bool(image_url) has_image = bool(image_url)
# Try to parse dates, but be lenient
date_debut, date_fin = datetime.now(), datetime.now() date_debut, date_fin = datetime.now(), datetime.now()
try: try:
date_debut, date_fin = parse_bonial_dates(dates_text) date_debut, date_fin = parse_bonial_dates(dates_text)
has_dates = True
except ValueError: except ValueError:
has_dates = False
# If we have URL and Image, we might still want it, but maybe mark as "dates unknown"
# For now, let's require dates or default to 1 week validity if title looks like a promo
if "semaine" in dates_text.lower() or "valable" in dates_text.lower(): if "semaine" in dates_text.lower() or "valable" in dates_text.lower():
date_fin = datetime.now().replace(day=datetime.now().day + 7) # Rough approx date_fin = datetime.now().replace(day=datetime.now().day + 7)
# STRICTER VALIDATION: Must have URL, Image AND (Dates OR strong title signal)
if has_url and has_image: if has_url and has_image:
catalogues.append({ catalogues.append({
"titre": titre.strip(), "titre": titre.strip(),
@@ -247,16 +183,10 @@ async def scrape_catalog_list(
}) })
logger.debug(f"Extracted catalog: {titre}") logger.debug(f"Extracted catalog: {titre}")
else: else:
# DEBUG: Log HTML content if extraction failed to help diagnosis if not has_url:
try: logger.warning(f"Skipping card {idx} (No URL): {titre}")
html_content = await card.inner_html() elif not has_image:
# Only log if it's not obviously bad logger.warning(f"Skipping card {idx} (No Image): {titre}")
if not has_url:
logger.warning(f"Skipping card {idx} (No URL): {titre}")
elif not has_image:
logger.warning(f"Skipping card {idx} (No Image): {titre}")
except Exception:
pass
except Exception as e: except Exception as e:
logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}") logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}")
@@ -267,36 +197,24 @@ async def scrape_catalog_list(
async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]: async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]:
""" """Scrape all pages of a catalog from the Bonial viewer."""
Scrape all pages of a catalog from the Bonial viewer.
Args:
page: Playwright page object
catalogue_url: URL of the catalog viewer
Returns:
List of page data dictionaries
"""
logger.info(f"Scraping catalog pages from {catalogue_url}") logger.info(f"Scraping catalog pages from {catalogue_url}")
await page.goto(catalogue_url, wait_until="networkidle") await page.goto(catalogue_url, wait_until="networkidle")
await asyncio.sleep(2) # Wait for viewer to initialize await asyncio.sleep(2)
# Try to close any popup/modal
try: try:
await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000) await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000)
await asyncio.sleep(1) await asyncio.sleep(1)
except Exception: except Exception:
pass pass
# Find all images from content-media.bonial.biz
images = await page.query_selector_all("img[src*='content-media.bonial.biz']") images = await page.query_selector_all("img[src*='content-media.bonial.biz']")
if not images: if not images:
logger.warning(f"No images found in catalog viewer: {catalogue_url}") logger.warning(f"No images found in catalog viewer: {catalogue_url}")
return [] return []
# Extract unique image URLs (filter duplicates and thumbnails)
seen_urls = set() seen_urls = set()
pages = [] pages = []
@@ -305,12 +223,9 @@ async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str,
src = await img.get_attribute("src") src = await img.get_attribute("src")
if src and src not in seen_urls: if src and src not in seen_urls:
# Filter out blurred/low-res images if possible
# Full res images are typically larger
width = await img.evaluate("el => el.naturalWidth") width = await img.evaluate("el => el.naturalWidth")
height = await img.evaluate("el => el.naturalHeight") height = await img.evaluate("el => el.naturalHeight")
# Skip very small images (likely thumbnails)
if width < 200 or height < 200: if width < 200 or height < 200:
continue continue
@@ -330,26 +245,12 @@ async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str,
return pages return pages
async def scrape_enseigne( async def scrape_enseigne(enseigne: Enseigne, db: Session, browser: Browser | None = None) -> ScrapingLog:
enseigne: Enseigne, """Scrape all catalogs for a specific enseigne."""
db: Session,
browser: Browser | None = None
) -> ScrapingLog:
"""
Scrape all catalogs for a specific enseigne.
Args:
enseigne: Enseigne to scrape
db: Database session
browser: Optional existing browser instance
Returns:
ScrapingLog object with execution details
"""
start_time = datetime.now() start_time = datetime.now()
log = ScrapingLog( log = ScrapingLog(
enseigne_id=enseigne.id, enseigne_id=enseigne.id,
statut="error", # Will be updated statut="error",
catalogues_trouves=0, catalogues_trouves=0,
catalogues_nouveaux=0, catalogues_nouveaux=0,
catalogues_mis_a_jour=0, catalogues_mis_a_jour=0,
@@ -358,19 +259,17 @@ async def scrape_enseigne(
own_browser = browser is None own_browser = browser is None
try: try:
# Create browser if not provided
if own_browser: if own_browser:
p = await async_playwright().start() browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000")
browser = await p.chromium.launch(headless=True) async with async_playwright() as p:
browser = await p.chromium.connect_over_cdp(browserless_url)
page = await browser.new_page() page = await browser.new_page()
# Set realistic user agent
await page.set_extra_http_headers({ await page.set_extra_http_headers({
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}) })
# Scrape catalog list
catalogues_data = await scrape_catalog_list(page, enseigne) catalogues_data = await scrape_catalog_list(page, enseigne)
log.catalogues_trouves = len(catalogues_data) log.catalogues_trouves = len(catalogues_data)
@@ -380,28 +279,23 @@ async def scrape_enseigne(
await page.close() await page.close()
return log return log
# Process each catalog
for cat_data in catalogues_data: for cat_data in catalogues_data:
try: try:
# Compute hash for duplicate detection
content_hash = compute_catalog_hash( content_hash = compute_catalog_hash(
enseigne.id, enseigne.id,
cat_data["titre"], cat_data["titre"],
cat_data["date_debut"] cat_data["date_debut"]
) )
# Check if catalog already exists
existing = db.query(Catalogue).filter_by(content_hash=content_hash).first() existing = db.query(Catalogue).filter_by(content_hash=content_hash).first()
if existing: if existing:
logger.debug(f"Catalog already exists: {cat_data['titre']}") logger.debug(f"Catalog already exists: {cat_data['titre']}")
continue continue
# Scrape catalog pages await asyncio.sleep(3)
await asyncio.sleep(3) # Respectful delay
pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"]) pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"])
# Create new catalog
catalogue = Catalogue( catalogue = Catalogue(
enseigne_id=enseigne.id, enseigne_id=enseigne.id,
titre=cat_data["titre"], titre=cat_data["titre"],
@@ -409,37 +303,36 @@ async def scrape_enseigne(
date_fin=cat_data["date_fin"], date_fin=cat_data["date_fin"],
image_couverture_url=cat_data["image_couverture_url"], image_couverture_url=cat_data["image_couverture_url"],
catalogue_url=cat_data["catalogue_url"], catalogue_url=cat_data["catalogue_url"],
statut="actif",
nombre_pages=len(pages_data), nombre_pages=len(pages_data),
content_hash=content_hash, content_hash=content_hash,
statut="actif" if cat_data["date_fin"] >= datetime.now() else "termine",
) )
db.add(catalogue) db.add(catalogue)
db.flush() # Get catalogue.id db.flush()
# Add pages
for page_data in pages_data: for page_data in pages_data:
cat_page = CataloguePage( catalogue_page = CataloguePage(
catalogue_id=catalogue.id, catalogue_id=catalogue.id,
**page_data numero_page=page_data["numero_page"],
image_url=page_data["image_url"],
largeur=page_data["largeur"],
hauteur=page_data["hauteur"],
) )
db.add(cat_page) db.add(catalogue_page)
log.catalogues_nouveaux += 1 log.catalogues_nouveaux += 1
logger.info(f"Created catalog: {cat_data['titre']} ({len(pages_data)} pages)") logger.info(f"Added new catalog: {cat_data['titre']}")
except Exception as e: except Exception as e:
logger.error(f"Error processing catalog {cat_data.get('titre')}: {e}") logger.error(f"Error processing catalog: {e}")
continue continue
db.commit()
log.statut = "success"
await page.close() await page.close()
# Update status
log.statut = "success" if log.catalogues_nouveaux > 0 else "partial"
db.commit()
except Exception as e: except Exception as e:
logger.error(f"Error scraping enseigne {enseigne.nom}: {e}") logger.error(f"Error scraping {enseigne.nom}: {e}")
log.statut = "error" log.statut = "error"
log.message_erreur = str(e) log.message_erreur = str(e)
db.rollback() db.rollback()
@@ -448,43 +341,29 @@ async def scrape_enseigne(
if own_browser and browser: if own_browser and browser:
await browser.close() await browser.close()
# Record duration log.duree_secondes = (datetime.now() - start_time).total_seconds()
log.duree_secondes = (datetime.now() - start_time).total_seconds() db.add(log)
db.add(log) db.commit()
db.commit()
return log return log
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]: async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
""" """Scrape all active enseignes. Uses Browserless service via WebSocket."""
Scrape all active enseignes.
Uses Browserless service via WebSocket.
Args:
db: Database session
Returns:
List of ScrapingLog objects
"""
import os
enseignes = db.query(Enseigne).filter_by(is_active=True).all() enseignes = db.query(Enseigne).filter_by(is_active=True).all()
logger.info(f"Starting scraping for {len(enseignes)} active enseignes") logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
logs = [] logs = []
# Get Browserless URL from environment
browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000") browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000")
logger.info(f"Connecting to Browserless at {browserless_url}") logger.info(f"Connecting to Browserless at {browserless_url}")
async with async_playwright() as p: async with async_playwright() as p:
# Connect to Browserless instead of launching local browser
browser = await p.chromium.connect_over_cdp(browserless_url) browser = await p.chromium.connect_over_cdp(browserless_url)
for enseigne in enseignes: for enseigne in enseignes:
try: try:
await asyncio.sleep(2) # Delay between enseignes await asyncio.sleep(2)
log = await scrape_enseigne(enseigne, db, browser) log = await scrape_enseigne(enseigne, db, browser)
logs.append(log) logs.append(log)
except Exception as e: except Exception as e: