feat: Implement Cataloguemate.fr scraper using BrowserlessService to replace Tiendeo and fetch promotional catalogs.

app/services/cataloguemate_scraper.py

@@ -3,6 +3,7 @@ Cataloguemate.fr Scraper Service
 
 Scrapes promotional catalogs from Cataloguemate.fr.
 Replaces Tiendeo scraper due to better reliability and simpler structure.
+Refactored to use BrowserlessService for robust containerized scraping.
 """
 
 import asyncio
@@ -12,16 +13,16 @@ from datetime import datetime, timedelta
 from typing import Any, Optional
 
 from bs4 import BeautifulSoup
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
 from sqlalchemy.orm import Session
 
-from app.models import Enseigne, Catalogue, ScrapingLog
+from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
+from app.services.browserless_service import browserless_service
 
 logger = logging.getLogger(__name__)
 
 BASE_URL = "https://www.cataloguemate.fr"
 
-async def scrape_catalog_list(crawler: AsyncWebCrawler, enseigne: Enseigne) -> list[dict[str, Any]]:
+async def scrape_catalog_list(enseigne: Enseigne) -> list[dict[str, Any]]:
     """
     Scrape the list of catalogs for an enseigne.
     We use a default city (Paris) because the main page often doesn't list catalogs directly.
@@ -38,24 +39,17 @@ async def scrape_catalog_list(crawler: AsyncWebCrawler, enseigne: Enseigne) -> l
     url = f"{BASE_URL}/offres/paris/{slug}/"
     logger.info(f"Scraping catalog list from: {url}")
 
-    config = CrawlerRunConfig(
+    # Use browserless service to get content
-        cache_mode=CacheMode.BYPASS,
+    html_content, _ = await browserless_service.get_page_content(url)
-        wait_for_images=True,
+    
-    )
+    if not html_content:
-
+        logger.error(f"Failed to fetch list {url}")
-    result = await crawler.arun(url=url, config=config)
-    if not result.success:
-        logger.error(f"Failed to crawl list {url}: {result.error_message}")
         return []
 
-    soup = BeautifulSoup(result.html, 'html.parser')
+    soup = BeautifulSoup(html_content, 'html.parser')
     catalogs = []
     
     # Find all links that might be catalogs
-    # On city pages, catalogs are often under "Ces produits sont actuellement dans les catalogues" 
-    # or "Catalogues populaires" or just listed.
-    # We look for links that look like /enseigne/titre-id/
-    
     links = soup.find_all('a', href=True)
     seen_urls = set()
     
@@ -104,7 +98,7 @@ async def scrape_catalog_list(crawler: AsyncWebCrawler, enseigne: Enseigne) -> l
     logger.info(f"Total catalogs found: {len(catalogs)}")
     return catalogs
 
-async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalog_url: str) -> list[dict[str, Any]]:
+async def scrape_catalog_pages(catalog_url: str) -> list[dict[str, Any]]:
     """
     Scrape pages from a specific catalog.
     Iterates through ?page=1, ?page=2 etc.
@@ -119,23 +113,16 @@ async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalog_url: str) -> li
         
         logger.info(f"Scraping page {page_num}: {current_url}")
         
-        config = CrawlerRunConfig(
+        # Use browserless service
-            cache_mode=CacheMode.BYPASS,
+        html_content, _ = await browserless_service.get_page_content(current_url)
-            wait_for_images=True,
-            delay_before_return_html=1.0
-        )
         
-        result = await crawler.arun(url=current_url, config=config)
+        if not html_content:
-        if not result.success:
             logger.warning(f"Failed to fetch page {page_num}")
             break
             
-        soup = BeautifulSoup(result.html, 'html.parser')
+        soup = BeautifulSoup(html_content, 'html.parser')
         
         # Find the main catalog image
-        # Strategy: Look for the largest image that contains 'page' or 'upload' in src
-        # or is inside a specific viewer container
-        
         main_image_url = None
         max_area = 0
         
@@ -162,14 +149,6 @@ async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalog_url: str) -> li
             # Check src for keywords
             is_likely_catalog = any(k in src.lower() for k in ['page', 'flyer', 'catalog', 'upload', 'images'])
             
-            # Calculate area
-            area = 0
-            if width and height:
-                try:
-                    area = int(width) * int(height)
-                except:
-                    pass
-            
             # Logic:
             # 1. If keyword match AND decent size -> Strong candidate
             # 2. If no keyword match but HUGE size -> Candidate
@@ -204,15 +183,11 @@ async def scrape_catalog_pages(crawler: AsyncWebCrawler, catalog_url: str) -> li
                 break
         
         # Check for pagination "Next" to know if we should continue
-        # Cataloguemate usually has a "Suivant" link or page numbers
-        # If we are on page X and there is no link to X+1, we stop
-        
-        # Simple check: if we found an image, try next page. 
-        # If the page loaded but had no image, we stop.
         if not main_image_url:
             break
             
         page_num += 1
+        # Small delay handled by browserless service already, but adding a tiny one here
         await asyncio.sleep(0.5)
 
     return pages
@@ -232,67 +207,54 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog:
     db.commit()
 
     try:
-        browser_config = BrowserConfig(headless=True)
+        # 0. Test connection (Implicit in get_page_content, but good to log)
-        async with AsyncWebCrawler(config=browser_config) as crawler:
+        logger.info(f"Starting scraping for {enseigne.nom} using Browserless...")
-            # 0. Test connection
-            logger.info("Testing browser connection...")
-            try:
-                test_result = await crawler.arun(url="https://www.cataloguemate.fr/", config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
-                if test_result.success:
-                    logger.info(f"Connection test successful (status: {len(test_result.html)} chars)")
-                else:
-                    logger.error(f"Connection test failed: {test_result.error_message}")
-                    raise Exception(f"Connection failed: {test_result.error_message}")
-            except Exception as e:
-                logger.error(f"Browser/Network error: {e}")
-                raise e
 
-            # 1. Get list of catalogs
+        # 1. Get list of catalogs
-            catalogs_list = await scrape_catalog_list(crawler, enseigne)
+        catalogs_list = await scrape_catalog_list(enseigne)
-            log.catalogues_trouves = len(catalogs_list)
+        log.catalogues_trouves = len(catalogs_list)
+        
+        for cat_info in catalogs_list:
+            # Check if catalog exists
+            existing = db.query(Catalogue).filter(
+                Catalogue.catalogue_url == cat_info['url']
+            ).first()
             
-            for cat_info in catalogs_list:
+            if existing:
-                # Check if catalog exists
+                continue
-                existing = db.query(Catalogue).filter(
-                    Catalogue.catalogue_url == cat_info['url']
-                ).first()
                 
-                if existing:
+            # 2. Scrape pages for new catalog
-                    continue
+            pages = await scrape_catalog_pages(cat_info['url'])
-                    
+            
-                # 2. Scrape pages for new catalog
+            if not pages:
-                pages = await scrape_catalog_pages(crawler, cat_info['url'])
+                continue
                 
-                if not pages:
+            # Create catalog entry
-                    continue
+            new_cat = Catalogue(
-                    
+                enseigne_id=enseigne.id,
-                # Create catalog entry
+                titre=cat_info['title'],
-                new_cat = Catalogue(
+                catalogue_url=cat_info['url'],
-                    enseigne_id=enseigne.id,
+                date_debut=datetime.now(), # Todo: extract real dates
-                    titre=cat_info['title'],
+                date_fin=datetime.now() + timedelta(days=14),
-                    catalogue_url=cat_info['url'],
+                image_couverture_url=pages[0]['image_url']
-                    date_debut=datetime.now(), # Todo: extract real dates
+            )
-                    date_fin=datetime.now() + timedelta(days=14),
+            db.add(new_cat)
-                    image_couverture_url=pages[0]['image_url']
+            db.commit()
+            db.refresh(new_cat)
+            
+            # Add pages
+            for page_data in pages:
+                new_page = CataloguePage(
+                    catalogue_id=new_cat.id,
+                    numero_page=page_data['page_number'],
+                    image_url=page_data['image_url'],
                 )
-                db.add(new_cat)
+                db.add(new_page)
-                db.commit()
+            
-                db.refresh(new_cat)
+            db.commit()
-                
+            log.catalogues_nouveaux += 1
-                # Add pages
+            logger.info(f"Saved catalog '{new_cat.titre}' with {len(pages)} pages")
-                for page_data in pages:
+            
-                    new_page = CataloguePage(
-                        catalogue_id=new_cat.id,
-                        numero_page=page_data['page_number'],
-                        image_url=page_data['image_url'],
-                        # We don't have dimensions from this method yet, but that's fine
-                    )
-                    db.add(new_page)
-                
-                db.commit()
-                log.catalogues_nouveaux += 1
-                logger.info(f"Saved catalog '{new_cat.titre}' with {len(pages)} pages")
-                
         log.statut = "success"
     except Exception as e:
         logger.error(f"Scraping failed: {e}")