From 79612690b34a79598d0df10f118aaa90d9e8fcf6 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 01:06:08 +0100 Subject: [PATCH] feat: add Tiendeo scraper service with Crawl4AI and update project dependencies and Dockerfile. --- Dockerfile | 30 +++ app/services/tiendeo_scraper.py | 346 +++++++++++++++++++------------- pyproject.toml | 1 + verify_crawl4ai_tiendeo.py | 164 +++++++++++++++ 4 files changed, 400 insertions(+), 141 deletions(-) create mode 100644 verify_crawl4ai_tiendeo.py diff --git a/Dockerfile b/Dockerfile index 2f74ea1..e5d3f13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,33 @@ WORKDIR /app # Enable unbuffered logging ENV PYTHONUNBUFFERED=1 +# Install system dependencies for crawl4ai and playwright +RUN apt-get update && apt-get install -y \ + wget \ + gnupg \ + # Playwright dependencies for Chromium + libnss3 \ + libnspr4 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libdbus-1-3 \ + libxkbcommon0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxrandr2 \ + libgbm1 \ + libasound2 \ + libpango-1.0-0 \ + libcairo2 \ + libatspi2.0-0 \ + fonts-liberation \ + libappindicator3-1 \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + # Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv @@ -29,6 +56,9 @@ COPY README.md . # Use uv for faster installation (fixed syntax) RUN uv pip install --system --no-cache . +# Install Playwright browsers for crawl4ai +RUN playwright install chromium --with-deps + # Copy backend code COPY app/ ./app COPY alembic.ini ./ diff --git a/app/services/tiendeo_scraper.py b/app/services/tiendeo_scraper.py index ec6aff4..25c584e 100644 --- a/app/services/tiendeo_scraper.py +++ b/app/services/tiendeo_scraper.py @@ -1,19 +1,19 @@ """ -Tiendeo.fr Scraper Service +Tiendeo.fr Scraper Service - Powered by Crawl4AI Scrapes promotional catalogs from Tiendeo.fr for configured enseignes. -Much simpler structure than Bonial - catalog cards are clearly defined with direct links. +Uses Crawl4AI for intelligent content extraction instead of fragile CSS selectors. """ import asyncio import hashlib import logging import re -import os from datetime import datetime, timedelta from typing import Any -from playwright.async_api import async_playwright, Page, Browser +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode +from pydantic import BaseModel, Field from sqlalchemy.orm import Session from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog @@ -27,6 +27,30 @@ TIENDEO_BASE_URL = "https://www.tiendeo.fr" DATE_PATTERN = r"(\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?)" +# ============================================================================ +# PYDANTIC SCHEMAS FOR EXTRACTION +# ============================================================================ + +class CatalogueCard(BaseModel): + """Schema for catalog card extraction""" + title: str = Field(description="Catalogue title") + url: str = Field(description="Full URL to the catalogue viewer") + image_url: str | None = Field(default=None, description="Cover image URL") + date_text: str | None = Field(default=None, description="Date validity text") + + +class CataloguePageSchema(BaseModel): + """Schema for catalog page image extraction""" + page_number: int = Field(description="Page number in sequence") + image_url: str = Field(description="Full resolution image URL") + width: int | None = Field(default=None, description="Image width in pixels") + height: int | None = Field(default=None, description="Image height in pixels") + + +# ============================================================================ +# DATE PARSING +# ============================================================================ + def parse_tiendeo_dates(date_str: str) -> tuple[datetime, datetime]: """ Parse Tiendeo date format to datetime objects. @@ -86,18 +110,34 @@ def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> return hashlib.sha256(content.encode()).hexdigest() -async def scrape_catalog_list(page: Page, enseigne: Enseigne) -> list[dict[str, Any]]: - """Scrape the list of catalogs for an enseigne from Tiendeo.""" +# ============================================================================ +# SCRAPING FUNCTIONS +# ============================================================================ + +async def scrape_catalog_list(crawler: AsyncWebCrawler, enseigne: Enseigne) -> list[dict[str, Any]]: + """Scrape the list of catalogs for an enseigne from Tiendeo using Crawl4AI.""" # Tiendeo uses store URLs like: /Magasins/{city}/{enseigne-slug} - # We'll use Nancy as default city for now url = f"{TIENDEO_BASE_URL}/Magasins/nancy/{enseigne.slug_bonial.lower()}" logger.info(f"Scraping catalog list for {enseigne.nom} from {url}") - await page.goto(url, wait_until="networkidle") - await asyncio.sleep(2) # Wait for dynamic content + # Configure crawler for this page + config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + wait_for_images=True, + process_iframes=False, + remove_overlay_elements=True, + wait_until="networkidle", + ) - # Extract catalog data using JavaScript - catalogs_data = await page.evaluate(""" + result = await crawler.arun(url=url, config=config) + + if not result.success: + logger.error(f"Failed to crawl {url}: {result.error_message}") + return [] + + # Extract catalog cards using JavaScript + catalog_data = await crawler.crawler_strategy.execute_js( + """ () => { const results = []; @@ -141,20 +181,20 @@ async def scrape_catalog_list(page: Page, enseigne: Enseigne) -> list[dict[str, url: href, image: imgSrc, containerText: containerText.substring(0, 300), // For date parsing - catalogId: catalogId }); } }); return results; } - """) + """ + ) - logger.info(f"Found {len(catalogs_data)} catalog links for {enseigne.nom}") + logger.info(f"Found {len(catalog_data)} catalog links for {enseigne.nom}") catalogues = [] - for idx, cat_data in enumerate(catalogs_data): + for idx, cat_data in enumerate(catalog_data): try: titre = cat_data.get('title', 'Catalogue') image_url = cat_data.get('image') @@ -165,7 +205,7 @@ async def scrape_catalog_list(page: Page, enseigne: Enseigne) -> list[dict[str, logger.warning(f"Skipping catalog {idx}: missing URL") continue - # Clean title - remove extra info like distances, "Gifi", etc. + # Clean title - remove extra info titre = titre.replace('Gifi', '').replace('Action', '').replace('Centrakor', '') titre = re.sub(r'\d+\.\d+\s*km', '', titre) # Remove "3.2 km" titre = re.sub(r'Nancy', '', titre, flags=re.IGNORECASE) @@ -206,53 +246,92 @@ async def scrape_catalog_list(page: Page, enseigne: Enseigne) -> list[dict[str, return catalogues -async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]: - """Scrape all pages of a catalog from the Tiendeo viewer.""" +async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalogue_url: str) -> list[dict[str, Any]]: + """Scrape all pages of a catalog from the Tiendeo viewer using Crawl4AI.""" logger.info(f"Scraping catalog pages from {catalogue_url}") - await page.goto(catalogue_url, wait_until="networkidle") - await asyncio.sleep(3) # Wait for viewer to load + # Configure crawler to wait for images + config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + wait_for_images=True, + process_iframes=True, + remove_overlay_elements=True, + wait_until="networkidle", + delay_before_return_html=3.0, # Wait for viewer to fully load + ) - # Find all catalog page images - images = await page.query_selector_all("img[src*='tiendeo'], img[src*='cdn']") + result = await crawler.arun(url=catalogue_url, config=config) - if not images: - logger.warning(f"No images found in catalog viewer: {catalogue_url}") + if not result.success: + logger.error(f"Failed to crawl {catalogue_url}: {result.error_message}") return [] - seen_urls = set() - pages = [] - - for idx, img in enumerate(images): - try: - src = await img.get_attribute("src") + # Extract catalog page images using JavaScript with intelligent filtering + pages_data = await crawler.crawler_strategy.execute_js( + """ + () => { + const results = []; + const seenUrls = new Set(); - if src and src not in seen_urls: - width = await img.evaluate("el => el.naturalWidth") - height = await img.evaluate("el => el.naturalHeight") + // Find all images - prioritize those in viewer containers + const allImages = document.querySelectorAll('img'); + + allImages.forEach((img) => { + // Get image source (try multiple attributes) + let src = img.src || img.getAttribute('data-src'); - # Skip small images (thumbnails) - if width < 200 or height < 200: - continue + // Try srcset as fallback + if (!src && img.srcset) { + const srcsetParts = img.srcset.split(',')[0].trim().split(' '); + src = srcsetParts[0]; + } - seen_urls.add(src) - pages.append({ - "numero_page": len(pages) + 1, - "image_url": src, - "largeur": width, - "hauteur": height, - }) - - except Exception as e: - logger.debug(f"Error processing image {idx}: {e}") - continue + if (!src || seenUrls.has(src)) return; + + // Skip obvious non-catalog images + if (src.includes('logo') || src.includes('icon') || src.includes('avatar')) { + return; + } + + // Get dimensions + const width = img.naturalWidth || img.width; + const height = img.naturalHeight || img.height; + + // Filter by dimensions - catalog pages are typically portrait and high-res + // Minimum 400px width, aspect ratio close to 3:4 or similar + if (width < 400 || height < 400) return; + + const aspectRatio = width / height; + + // Portrait images (0.5 to 0.9 ratio) - catalog pages are usually portrait + if (aspectRatio > 0.5 && aspectRatio < 0.9) { + seenUrls.add(src); + results.push({ + image_url: src, + width: width, + height: height, + }); + } + }); + + // Sort by area (largest first) to prioritize full resolution images + results.sort((a, b) => (b.width * b.height) - (a.width * a.height)); + + // Assign page numbers + return results.map((item, index) => ({ + ...item, + numero_page: index + 1, + })); + } + """ + ) - logger.info(f"Found {len(pages)} pages in catalog") - return pages + logger.info(f"Found {len(pages_data)} pages in catalog") + return pages_data -async def scrape_enseigne(enseigne: Enseigne, db: Session, browser: Browser | None = None) -> ScrapingLog: - """Scrape all catalogs for a specific enseigne from Tiendeo.""" +async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog: + """Scrape all catalogs for a specific enseigne from Tiendeo using Crawl4AI.""" start_time = datetime.now() log = ScrapingLog( enseigne_id=enseigne.id, @@ -262,80 +341,77 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session, browser: Browser | No catalogues_mis_a_jour=0, ) - own_browser = browser is None - try: - if own_browser: - browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000") - async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp(browserless_url) + # Configure browser for Crawl4AI + browser_config = BrowserConfig( + headless=True, + verbose=False, + extra_args=["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"], + ) - page = await browser.new_page() - - await page.set_extra_http_headers({ - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - }) - - catalogues_data = await scrape_catalog_list(page, enseigne) - log.catalogues_trouves = len(catalogues_data) - - if not catalogues_data: - log.statut = "success" - log.message_erreur = "No catalogs found (may be normal)" - await page.close() - return log - - for cat_data in catalogues_data: - try: - content_hash = compute_catalog_hash( - enseigne.id, - cat_data["titre"], - cat_data["date_debut"] - ) - - existing = db.query(Catalogue).filter_by(content_hash=content_hash).first() - - if existing: - logger.debug(f"Catalog already exists: {cat_data['titre']}") - continue - - await asyncio.sleep(2) # Respectful delay - pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"]) - - catalogue = Catalogue( - enseigne_id=enseigne.id, - titre=cat_data["titre"], - date_debut=cat_data["date_debut"], - date_fin=cat_data["date_fin"], - image_couverture_url=cat_data["image_couverture_url"], - catalogue_url=cat_data["catalogue_url"], - statut="actif", - nombre_pages=len(pages_data), - content_hash=content_hash, - ) - db.add(catalogue) - db.flush() - - for page_data in pages_data: - catalogue_page = CataloguePage( - catalogue_id=catalogue.id, - numero_page=page_data["numero_page"], - image_url=page_data["image_url"], - largeur=page_data["largeur"], - hauteur=page_data["hauteur"], + async with AsyncWebCrawler(config=browser_config) as crawler: + catalogues_data = await scrape_catalog_list(crawler, enseigne) + log.catalogues_trouves = len(catalogues_data) + + if not catalogues_data: + log.statut = "success" + log.message_erreur = "No catalogs found (may be normal)" + return log + + for cat_data in catalogues_data: + try: + content_hash = compute_catalog_hash( + enseigne.id, + cat_data["titre"], + cat_data["date_debut"] ) - db.add(catalogue_page) - - log.catalogues_nouveaux += 1 - logger.info(f"Added new catalog: {cat_data['titre']}") - - except Exception as e: - logger.error(f"Error processing catalog: {e}") - continue - - db.commit() - log.statut = "success" - await page.close() + + existing = db.query(Catalogue).filter_by(content_hash=content_hash).first() + + if existing: + logger.debug(f"Catalog already exists: {cat_data['titre']}") + continue + + await asyncio.sleep(2) # Respectful delay + pages_data = await scrape_catalog_pages(crawler, cat_data["catalogue_url"]) + + if not pages_data: + logger.warning(f"No pages found for catalog: {cat_data['titre']}") + continue + + catalogue = Catalogue( + enseigne_id=enseigne.id, + titre=cat_data["titre"], + date_debut=cat_data["date_debut"], + date_fin=cat_data["date_fin"], + image_couverture_url=cat_data["image_couverture_url"], + catalogue_url=cat_data["catalogue_url"], + statut="actif", + nombre_pages=len(pages_data), + content_hash=content_hash, + ) + db.add(catalogue) + db.flush() + + for page_data in pages_data: + catalogue_page = CataloguePage( + catalogue_id=catalogue.id, + numero_page=page_data["numero_page"], + image_url=page_data["image_url"], + largeur=page_data.get("width"), + hauteur=page_data.get("height"), + ) + db.add(catalogue_page) + + log.catalogues_nouveaux += 1 + logger.info(f"Added new catalog: {cat_data['titre']} with {len(pages_data)} pages") + + except Exception as e: + logger.error(f"Error processing catalog: {e}") + continue + + db.commit() + log.statut = "success" except Exception as e: logger.error(f"Error scraping {enseigne.nom}: {e}") @@ -343,10 +419,6 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session, browser: Browser | No log.message_erreur = str(e) db.rollback() - finally: - if own_browser and browser: - await browser.close() - log.duree_secondes = (datetime.now() - start_time).total_seconds() db.add(log) db.commit() @@ -361,21 +433,13 @@ async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]: logs = [] - browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000") - logger.info(f"Connecting to Browserless at {browserless_url}") - - async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp(browserless_url) - - for enseigne in enseignes: - try: - await asyncio.sleep(2) - log = await scrape_enseigne(enseigne, db, browser) - logs.append(log) - except Exception as e: - logger.error(f"Failed to scrape {enseigne.nom}: {e}") - - await browser.close() + for enseigne in enseignes: + try: + await asyncio.sleep(2) + log = await scrape_enseigne(enseigne, db) + logs.append(log) + except Exception as e: + logger.error(f"Failed to scrape {enseigne.nom}: {e}") logger.info(f"Tiendeo scraping complete: {len(logs)} enseignes processed") return logs diff --git a/pyproject.toml b/pyproject.toml index 1c9156d..9fd2872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "beautifulsoup4", "lxml", "pyjwt", + "crawl4ai", ] [build-system] diff --git a/verify_crawl4ai_tiendeo.py b/verify_crawl4ai_tiendeo.py new file mode 100644 index 0000000..9e0d2f6 --- /dev/null +++ b/verify_crawl4ai_tiendeo.py @@ -0,0 +1,164 @@ +""" +Script de vérification pour tester Crawl4AI dans le scraper Tiendeo. + +Usage: + python verify_crawl4ai_scraper.py + +Ce script teste: +1. Import de Crawl4AI +2. Extraction d'une page catalogue Tiendeo +3. Comptage des pages trouvées +""" + +import asyncio +import logging +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +async def test_crawl4ai_import(): + """Test 1: Vérifier que Crawl4AI est bien installé""" + try: + logger.info("✓ Crawl4AI importé avec succès") + logger.info(f" Version: {AsyncWebCrawler.__module__}") + return True + except Exception as e: + logger.error(f"✗ Erreur import Crawl4AI: {e}") + return False + + +async def test_catalog_page_extraction(): + """Test 2: Extraire les pages d'un catalogue Tiendeo""" + # URL de test - catalogue Gifi Nancy (à adapter si nécessaire) + test_url = "https://www.tiendeo.fr/Catalogues/nancy/gifi" + + logger.info(f"\nTest extraction depuis: {test_url}") + + try: + browser_config = BrowserConfig( + headless=True, + verbose=False, + extra_args=["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"], + ) + + config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + wait_for_images=True, + process_iframes=True, + remove_overlay_elements=True, + wait_until="networkidle", + delay_before_return_html=3.0, + ) + + async with AsyncWebCrawler(config=browser_config) as crawler: + result = await crawler.arun(url=test_url, config=config) + + if not result.success: + logger.error(f"✗ Échec du crawling: {result.error_message}") + return False + + logger.info(f"✓ Page chargée avec succès") + logger.info(f" HTML length: {len(result.html)} chars") + + # Tester l'extraction JavaScript + pages_data = await crawler.crawler_strategy.execute_js( + """ + () => { + const results = []; + const seenUrls = new Set(); + + const allImages = document.querySelectorAll('img'); + + allImages.forEach((img) => { + let src = img.src || img.getAttribute('data-src'); + + if (!src && img.srcset) { + const srcsetParts = img.srcset.split(',')[0].trim().split(' '); + src = srcsetParts[0]; + } + + if (!src || seenUrls.has(src)) return; + + if (src.includes('logo') || src.includes('icon') || src.includes('avatar')) { + return; + } + + const width = img.naturalWidth || img.width; + const height = img.naturalHeight || img.height; + + if (width < 400 || height < 400) return; + + const aspectRatio = width / height; + + if (aspectRatio > 0.5 && aspectRatio < 0.9) { + seenUrls.add(src); + results.push({ + image_url: src, + width: width, + height: height, + }); + } + }); + + results.sort((a, b) => (b.width * b.height) - (a.width * a.height)); + + return results.map((item, index) => ({ + ...item, + numero_page: index + 1, + })); + } + """ + ) + + logger.info(f"✓ Extraction JavaScript réussie") + logger.info(f" Pages trouvées: {len(pages_data)}") + + if len(pages_data) > 0: + logger.info(f"\n Exemples de pages:") + for page in pages_data[:3]: + logger.info(f" - Page {page['numero_page']}: {page['width']}x{page['height']}px") + logger.info(f" URL: {page['image_url'][:80]}...") + return True + else: + logger.warning("⚠ Aucune page trouvée (structure du site a peut-être changé)") + return False + + except Exception as e: + logger.error(f"✗ Erreur lors de l'extraction: {e}") + import traceback + traceback.print_exc() + return False + + +async def main(): + """Exécuter tous les tests""" + logger.info("=" * 70) + logger.info("VÉRIFICATION CRAWL4AI - SCRAPER TIENDEO") + logger.info("=" * 70) + + # Test 1: Import + test1 = await test_crawl4ai_import() + + # Test 2: Extraction + test2 = await test_catalog_page_extraction() + + # Résumé + logger.info("\n" + "=" * 70) + logger.info("RÉSUMÉ") + logger.info("=" * 70) + logger.info(f"Import Crawl4AI: {'✓ OK' if test1 else '✗ ÉCHEC'}") + logger.info(f"Extraction pages: {'✓ OK' if test2 else '✗ ÉCHEC'}") + + if test1 and test2: + logger.info("\n✓ TOUS LES TESTS SONT PASSÉS!") + logger.info("Le scraper Crawl4AI est prêt à être utilisé.") + else: + logger.info("\n✗ CERTAINS TESTS ONT ÉCHOUÉ") + logger.info("Vérifiez les erreurs ci-dessus.") + + +if __name__ == "__main__": + asyncio.run(main())