mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-14 10:46:32 +02:00
536 lines
19 KiB
Python
536 lines
19 KiB
Python
"""
|
|
Tiendeo.fr Scraper Service - Powered by Crawl4AI
|
|
|
|
Scrapes promotional catalogs from Tiendeo.fr for configured enseignes.
|
|
Uses Crawl4AI for intelligent content extraction instead of fragile CSS selectors.
|
|
"""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timedelta
|
|
from typing import Any
|
|
|
|
from bs4 import BeautifulSoup
|
|
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Tiendeo site base URL
|
|
TIENDEO_BASE_URL = "https://www.tiendeo.fr"
|
|
|
|
# Date pattern for Tiendeo: "Expire le 31/12" or "mar. 25/11 - lun. 08/12"
|
|
DATE_PATTERN = r"(\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)"
|
|
|
|
|
|
# ============================================================================
|
|
# PYDANTIC SCHEMAS FOR EXTRACTION
|
|
# ============================================================================
|
|
|
|
class CatalogueCard(BaseModel):
|
|
"""Schema for catalog card extraction"""
|
|
title: str = Field(description="Catalogue title")
|
|
url: str = Field(description="Full URL to the catalogue viewer")
|
|
image_url: str | None = Field(default=None, description="Cover image URL")
|
|
date_text: str | None = Field(default=None, description="Date validity text")
|
|
|
|
|
|
class CataloguePageSchema(BaseModel):
|
|
"""Schema for catalog page image extraction"""
|
|
page_number: int = Field(description="Page number in sequence")
|
|
image_url: str = Field(description="Full resolution image URL")
|
|
width: int | None = Field(default=None, description="Image width in pixels")
|
|
height: int | None = Field(default=None, description="Image height in pixels")
|
|
|
|
|
|
# ============================================================================
|
|
# DATE PARSING
|
|
# ============================================================================
|
|
|
|
def parse_tiendeo_dates(date_str: str) -> tuple[datetime, datetime]:
|
|
"""
|
|
Parse Tiendeo date format to datetime objects.
|
|
|
|
Args:
|
|
date_str: Date string like "Expire le 31/12" or "mar. 25/11 - lun. 08/12"
|
|
|
|
Returns:
|
|
Tuple of (date_debut, date_fin)
|
|
"""
|
|
matches = re.findall(DATE_PATTERN, date_str)
|
|
|
|
if not matches:
|
|
logger.warning(f"No dates found in '{date_str}', using default duration")
|
|
raise ValueError(f"Invalid date format: {date_str}")
|
|
|
|
today = datetime.now()
|
|
current_year = today.year
|
|
|
|
parsed_dates = []
|
|
for date_match in matches:
|
|
date_match = date_match.replace("-", "/")
|
|
parts = date_match.split("/")
|
|
|
|
day = int(parts[0])
|
|
month = int(parts[1])
|
|
year = int(parts[2]) if len(parts) > 2 else current_year
|
|
|
|
if year < 100:
|
|
year += 2000
|
|
|
|
# Handle year rollover
|
|
if len(parts) == 2:
|
|
if month < today.month - 3:
|
|
year += 1
|
|
|
|
try:
|
|
parsed_dates.append(datetime(year, month, day))
|
|
except ValueError:
|
|
continue
|
|
|
|
if not parsed_dates:
|
|
raise ValueError("No valid dates parsed")
|
|
|
|
if len(parsed_dates) >= 2:
|
|
return min(parsed_dates), max(parsed_dates)
|
|
elif len(parsed_dates) == 1:
|
|
# If only one date (expire date), use today as start
|
|
return today, parsed_dates[0]
|
|
|
|
return parsed_dates[0], parsed_dates[0]
|
|
|
|
|
|
def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str:
|
|
"""Compute SHA256 hash for duplicate detection."""
|
|
content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}"
|
|
return hashlib.sha256(content.encode()).hexdigest()
|
|
|
|
|
|
# ============================================================================
|
|
# SCRAPING FUNCTIONS
|
|
# ============================================================================
|
|
|
|
async def scrape_catalog_list(crawler: AsyncWebCrawler, enseigne: Enseigne) -> list[dict[str, Any]]:
|
|
"""Scrape the list of catalogs for an enseigne from Tiendeo using Crawl4AI."""
|
|
# Tiendeo uses store URLs like: /Magasins/{city}/{enseigne-slug}
|
|
url = f"{TIENDEO_BASE_URL}/Magasins/nancy/{enseigne.slug_bonial.lower()}"
|
|
logger.info(f"Scraping catalog list for {enseigne.nom} from {url}")
|
|
|
|
# JavaScript to extract catalog data
|
|
extraction_js = """
|
|
const results = [];
|
|
|
|
// Find all links to catalog pages (format: /Catalogues/{id})
|
|
const catalogLinks = document.querySelectorAll('a[href*="/Catalogues/"]');
|
|
|
|
catalogLinks.forEach((link) => {
|
|
const href = link.href;
|
|
const catalogId = href.split('/Catalogues/')[1];
|
|
|
|
if (!catalogId) return;
|
|
|
|
// Extract title from link text or nearby heading
|
|
let title = link.textContent.trim();
|
|
|
|
// Try to find h3 or h4 near this link for better title
|
|
const parentContainer = link.closest('div, section, article');
|
|
if (parentContainer) {
|
|
const heading = parentContainer.querySelector('h3, h4, h2');
|
|
if (heading && heading.textContent.trim().length > title.length) {
|
|
title = heading.textContent.trim();
|
|
}
|
|
}
|
|
|
|
// Look for image in the same container
|
|
let imgSrc = null;
|
|
if (parentContainer) {
|
|
const img = parentContainer.querySelector('img');
|
|
if (img) {
|
|
imgSrc = img.src || img.getAttribute('data-src') || img.getAttribute('srcset')?.split(' ')[0];
|
|
}
|
|
}
|
|
|
|
// Extract date information from container text
|
|
const containerText = parentContainer ? parentContainer.textContent : link.textContent;
|
|
|
|
// Avoid duplicates
|
|
if (!results.some(r => r.url === href)) {
|
|
results.push({
|
|
title: title,
|
|
url: href,
|
|
image: imgSrc,
|
|
containerText: containerText.substring(0, 300), // For date parsing
|
|
});
|
|
}
|
|
});
|
|
|
|
// Store results in window for retrieval
|
|
window.__catalogData = results;
|
|
"""
|
|
|
|
# Configure crawler for this page with JavaScript extraction
|
|
config = CrawlerRunConfig(
|
|
cache_mode=CacheMode.BYPASS,
|
|
wait_for_images=True,
|
|
process_iframes=False,
|
|
remove_overlay_elements=True,
|
|
wait_until="networkidle",
|
|
js_code=extraction_js, # Execute extraction JavaScript
|
|
)
|
|
|
|
result = await crawler.arun(url=url, config=config)
|
|
|
|
if not result.success:
|
|
logger.error(f"Failed to crawl {url}: {result.error_message}")
|
|
return []
|
|
|
|
# Parse the HTML to extract data (fallback if js_code doesn't return data directly)
|
|
# We'll use BeautifulSoup as a reliable method
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(result.html, 'html.parser')
|
|
|
|
catalog_links = soup.find_all('a', href=lambda h: h and '/Catalogues/' in h)
|
|
|
|
catalog_data = []
|
|
seen_urls = set()
|
|
|
|
for link in catalog_links:
|
|
href = link.get('href', '')
|
|
if not href or href in seen_urls:
|
|
continue
|
|
|
|
# Make absolute URL
|
|
if href.startswith('/'):
|
|
href = TIENDEO_BASE_URL + href
|
|
|
|
catalog_id = href.split('/Catalogues/')[-1].split('/')[0] if '/Catalogues/' in href else None
|
|
if not catalog_id:
|
|
continue
|
|
|
|
# CRITICAL FIX: Filter by enseigne name
|
|
# The catalog link or its parent container MUST contain the enseigne name
|
|
link_text = link.get_text(strip=True)
|
|
parent = link.find_parent(['div', 'section', 'article', 'li'])
|
|
parent_text = parent.get_text(strip=True) if parent else ''
|
|
|
|
# Combined text to search in
|
|
combined_text = f"{link_text} {parent_text}".lower()
|
|
enseigne_name_lower = enseigne.nom.lower()
|
|
|
|
# Check if enseigne name is in the link or parent text
|
|
if enseigne_name_lower not in combined_text:
|
|
logger.debug(f"Skipping catalog (not for {enseigne.nom}): {link_text[:50]}")
|
|
continue
|
|
|
|
seen_urls.add(href)
|
|
|
|
# Extract title
|
|
title = link.get_text(strip=True)
|
|
|
|
# Try to find better title in parent container
|
|
parent = link.find_parent(['div', 'section', 'article'])
|
|
if parent:
|
|
heading = parent.find(['h2', 'h3', 'h4'])
|
|
if heading and len(heading.get_text(strip=True)) > len(title):
|
|
title = heading.get_text(strip=True)
|
|
|
|
# Find image
|
|
img = parent.find('img')
|
|
img_src = None
|
|
if img:
|
|
img_src = img.get('src') or img.get('data-src')
|
|
if not img_src and img.get('srcset'):
|
|
img_src = img.get('srcset').split(' ')[0]
|
|
|
|
container_text = parent.get_text(strip=True)[:300]
|
|
else:
|
|
img_src = None
|
|
container_text = title
|
|
|
|
catalog_data.append({
|
|
'title': title,
|
|
'url': href,
|
|
'image': img_src,
|
|
'containerText': container_text,
|
|
})
|
|
|
|
logger.info(f"Found {len(catalog_data)} catalog links for {enseigne.nom}")
|
|
|
|
catalogues = []
|
|
|
|
for idx, cat_data in enumerate(catalog_data):
|
|
try:
|
|
titre = cat_data.get('title', 'Catalogue')
|
|
image_url = cat_data.get('image')
|
|
catalogue_url = cat_data.get('url')
|
|
container_text = cat_data.get('containerText', '')
|
|
|
|
if not catalogue_url:
|
|
logger.warning(f"Skipping catalog {idx}: missing URL")
|
|
continue
|
|
|
|
# Clean title - remove extra info
|
|
titre = titre.replace('Gifi', '').replace('Action', '').replace('Centrakor', '')
|
|
titre = re.sub(r'\d+\.\d+\s*km', '', titre) # Remove "3.2 km"
|
|
titre = re.sub(r'Nancy', '', titre, flags=re.IGNORECASE)
|
|
titre = re.sub(r'Expire\s+le.*', '', titre) # Remove expire info from title
|
|
titre = titre.strip()
|
|
|
|
# Skip if title is empty or too short
|
|
if len(titre) < 3:
|
|
logger.warning(f"Skipping catalog {idx}: title too short after cleaning")
|
|
continue
|
|
|
|
# Ensure absolute URL
|
|
if image_url and image_url.startswith('//'):
|
|
image_url = 'https:' + image_url
|
|
|
|
# Parse dates from container text
|
|
date_debut, date_fin = datetime.now(), datetime.now() + timedelta(days=30)
|
|
try:
|
|
date_debut, date_fin = parse_tiendeo_dates(container_text)
|
|
except ValueError:
|
|
# Default to 30 days validity
|
|
logger.debug(f"Could not parse dates for '{titre}', using 30 days default")
|
|
|
|
catalogues.append({
|
|
"titre": titre,
|
|
"date_debut": date_debut,
|
|
"date_fin": date_fin,
|
|
"image_couverture_url": image_url or "",
|
|
"catalogue_url": catalogue_url,
|
|
})
|
|
logger.info(f"Extracted catalog: {titre}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing catalog {idx} for {enseigne.nom}: {e}")
|
|
continue
|
|
|
|
logger.info(f"Successfully extracted {len(catalogues)} catalogs for {enseigne.nom}")
|
|
return catalogues
|
|
|
|
|
|
async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalogue_url: str) -> list[dict[str, Any]]:
|
|
"""Scrape all pages of a catalog from the Tiendeo viewer using Crawl4AI."""
|
|
logger.info(f"Scraping catalog pages from {catalogue_url}")
|
|
|
|
# JavaScript to trigger loading of all pages in the viewer
|
|
load_all_pages_js = """
|
|
// Scroll down to trigger lazy loading of catalog pages
|
|
window.scrollTo(0, document.body.scrollHeight);
|
|
|
|
// Wait a bit for images to start loading
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
|
|
// Try to find and click "next page" buttons to load more pages
|
|
const nextButtons = document.querySelectorAll('[class*="next"], [class*="Next"], button[aria-label*="next" i]');
|
|
for (let btn of nextButtons) {
|
|
if (btn && btn.offsetParent !== null) { // visible
|
|
btn.click();
|
|
await new Promise(r => setTimeout(r, 500));
|
|
}
|
|
}
|
|
|
|
// Scroll again after clicking
|
|
window.scrollTo(0, document.body.scrollHeight);
|
|
"""
|
|
|
|
# Configure crawler to wait for images and execute JS
|
|
config = CrawlerRunConfig(
|
|
cache_mode=CacheMode.BYPASS,
|
|
wait_for_images=True,
|
|
process_iframes=True,
|
|
remove_overlay_elements=True,
|
|
wait_until="networkidle",
|
|
delay_before_return_html=5.0, # Wait longer for viewer to fully load
|
|
js_code=load_all_pages_js, # Execute JS to load all pages
|
|
)
|
|
|
|
result = await crawler.arun(url=catalogue_url, config=config)
|
|
|
|
if not result.success:
|
|
logger.error(f"Failed to crawl {catalogue_url}: {result.error_message}")
|
|
return []
|
|
|
|
# Parse HTML to extract catalog page images
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(result.html, 'html.parser')
|
|
|
|
all_images = soup.find_all('img')
|
|
|
|
pages_data = []
|
|
seen_urls = set()
|
|
|
|
for img in all_images:
|
|
# Get image source (try multiple attributes)
|
|
src = img.get('src') or img.get('data-src')
|
|
|
|
# Try srcset as fallback
|
|
if not src and img.get('srcset'):
|
|
srcset_parts = img.get('srcset').split(',')[0].strip().split(' ')
|
|
src = srcset_parts[0]
|
|
|
|
if not src or src in seen_urls:
|
|
continue
|
|
|
|
# Skip obvious non-catalog images
|
|
if 'logo' in src.lower() or 'icon' in src.lower() or 'avatar' in src.lower():
|
|
continue
|
|
|
|
# Get dimensions from attributes (not perfect but workable fallback)
|
|
width = img.get('width')
|
|
height = img.get('height')
|
|
|
|
# Try to parse as int
|
|
try:
|
|
width = int(width) if width else 800 # Default assumption for catalog pages
|
|
height = int(height) if height else 1100 # Default portrait ratio
|
|
except (ValueError, TypeError):
|
|
width = 800
|
|
height = 1100
|
|
|
|
# Filter by dimensions - catalog pages are typically portrait and high-res
|
|
if width < 400 or height < 400:
|
|
continue
|
|
|
|
aspect_ratio = width / height
|
|
|
|
# Portrait images (0.5 to 0.9 ratio) - catalog pages are usually portrait
|
|
if 0.5 < aspect_ratio < 0.9:
|
|
seen_urls.add(src)
|
|
pages_data.append({
|
|
'image_url': src,
|
|
'width': width,
|
|
'height': height,
|
|
})
|
|
|
|
# Sort by area (largest first) to prioritize full resolution images
|
|
pages_data.sort(key=lambda x: x['width'] * x['height'], reverse=True)
|
|
|
|
# Assign page numbers
|
|
for idx, page in enumerate(pages_data):
|
|
page['numero_page'] = idx + 1
|
|
|
|
logger.info(f"Found {len(pages_data)} pages in catalog")
|
|
return pages_data
|
|
|
|
|
|
async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog:
|
|
"""Scrape all catalogs for a specific enseigne from Tiendeo using Crawl4AI."""
|
|
start_time = datetime.now()
|
|
log = ScrapingLog(
|
|
enseigne_id=enseigne.id,
|
|
statut="error",
|
|
catalogues_trouves=0,
|
|
catalogues_nouveaux=0,
|
|
catalogues_mis_a_jour=0,
|
|
)
|
|
|
|
try:
|
|
# Configure browser for Crawl4AI
|
|
browser_config = BrowserConfig(
|
|
headless=True,
|
|
verbose=False,
|
|
extra_args=["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
|
|
)
|
|
|
|
async with AsyncWebCrawler(config=browser_config) as crawler:
|
|
catalogues_data = await scrape_catalog_list(crawler, enseigne)
|
|
log.catalogues_trouves = len(catalogues_data)
|
|
|
|
if not catalogues_data:
|
|
log.statut = "success"
|
|
log.message_erreur = "No catalogs found (may be normal)"
|
|
return log
|
|
|
|
for cat_data in catalogues_data:
|
|
try:
|
|
content_hash = compute_catalog_hash(
|
|
enseigne.id,
|
|
cat_data["titre"],
|
|
cat_data["date_debut"]
|
|
)
|
|
|
|
existing = db.query(Catalogue).filter_by(content_hash=content_hash).first()
|
|
|
|
if existing:
|
|
logger.debug(f"Catalog already exists: {cat_data['titre']}")
|
|
continue
|
|
|
|
await asyncio.sleep(2) # Respectful delay
|
|
pages_data = await scrape_catalog_pages(crawler, cat_data["catalogue_url"])
|
|
|
|
if not pages_data:
|
|
logger.warning(f"No pages found for catalog: {cat_data['titre']}")
|
|
continue
|
|
|
|
catalogue = Catalogue(
|
|
enseigne_id=enseigne.id,
|
|
titre=cat_data["titre"],
|
|
date_debut=cat_data["date_debut"],
|
|
date_fin=cat_data["date_fin"],
|
|
image_couverture_url=cat_data["image_couverture_url"],
|
|
catalogue_url=cat_data["catalogue_url"],
|
|
statut="actif",
|
|
nombre_pages=len(pages_data),
|
|
content_hash=content_hash,
|
|
)
|
|
db.add(catalogue)
|
|
db.flush()
|
|
|
|
for page_data in pages_data:
|
|
catalogue_page = CataloguePage(
|
|
catalogue_id=catalogue.id,
|
|
numero_page=page_data["numero_page"],
|
|
image_url=page_data["image_url"],
|
|
largeur=page_data.get("width"),
|
|
hauteur=page_data.get("height"),
|
|
)
|
|
db.add(catalogue_page)
|
|
|
|
log.catalogues_nouveaux += 1
|
|
logger.info(f"Added new catalog: {cat_data['titre']} with {len(pages_data)} pages")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing catalog: {e}")
|
|
continue
|
|
|
|
db.commit()
|
|
log.statut = "success"
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error scraping {enseigne.nom}: {e}")
|
|
log.statut = "error"
|
|
log.message_erreur = str(e)
|
|
db.rollback()
|
|
|
|
log.duree_secondes = (datetime.now() - start_time).total_seconds()
|
|
db.add(log)
|
|
db.commit()
|
|
|
|
return log
|
|
|
|
|
|
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
|
"""Scrape all active enseignes from Tiendeo."""
|
|
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
|
|
logger.info(f"Starting Tiendeo scraping for {len(enseignes)} active enseignes")
|
|
|
|
logs = []
|
|
|
|
for enseigne in enseignes:
|
|
try:
|
|
await asyncio.sleep(2)
|
|
log = await scrape_enseigne(enseigne, db)
|
|
logs.append(log)
|
|
except Exception as e:
|
|
logger.error(f"Failed to scrape {enseigne.nom}: {e}")
|
|
|
|
logger.info(f"Tiendeo scraping complete: {len(logs)} enseignes processed")
|
|
return logs
|