feat: Add initial application structure, core data models, and a new catalog module with Bonial scraping capabilities.

app/alembic/versions/c1d2e3f4g5h6_add_catalog_module.py

@@ -0,0 +1,103 @@
+"""Add catalog module tables
+
+Revision ID: c1d2e3f4g5h6
+Revises: b2c3d4e5f6g7
+Create Date: 2025-11-29 18:56:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c1d2e3f4g5h6"
+down_revision: str | None = "b2c3d4e5f6g7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Create tables for the catalog module."""
+    
+    # Create enseignes table
+    op.create_table(
+        "enseignes",
+        sa.Column("id", sa.Integer(), primary_key=True, index=True),
+        sa.Column("nom", sa.String(), nullable=False),
+        sa.Column("slug_bonial", sa.String(), nullable=False, unique=True, index=True),
+        sa.Column("logo_url", sa.String(), nullable=True),
+        sa.Column("couleur", sa.String(), nullable=False),
+        sa.Column("site_url", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("is_active", sa.Boolean(), default=True),
+        sa.Column("ordre_affichage", sa.Integer(), default=0),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+    )
+    
+    # Create catalogues table
+    op.create_table(
+        "catalogues",
+        sa.Column("id", sa.Integer(), primary_key=True, index=True),
+        sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=False, index=True),
+        sa.Column("titre", sa.String(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("date_debut", sa.DateTime(), nullable=False, index=True),
+        sa.Column("date_fin", sa.DateTime(), nullable=False, index=True),
+        sa.Column("image_couverture_url", sa.String(), nullable=False),
+        sa.Column("catalogue_url", sa.String(), nullable=False),
+        sa.Column("nombre_pages", sa.Integer(), default=0),
+        sa.Column("statut", sa.String(), default="actif", index=True),
+        sa.Column("content_hash", sa.String(64), nullable=False, unique=True, index=True),
+        sa.Column("metadonnees", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+    )
+    
+    # Create catalogue_pages table
+    op.create_table(
+        "catalogue_pages",
+        sa.Column("id", sa.Integer(), primary_key=True, index=True),
+        sa.Column("catalogue_id", sa.Integer(), sa.ForeignKey("catalogues.id"), nullable=False, index=True),
+        sa.Column("numero_page", sa.Integer(), nullable=False),
+        sa.Column("image_url", sa.String(), nullable=False),
+        sa.Column("image_thumbnail_url", sa.String(), nullable=True),
+        sa.Column("largeur", sa.Integer(), nullable=True),
+        sa.Column("hauteur", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+    )
+    
+    # Create unique index on catalogue_id + numero_page
+    op.create_index(
+        "idx_catalogue_page_numero",
+        "catalogue_pages",
+        ["catalogue_id", "numero_page"],
+        unique=True,
+    )
+    
+    # Create scraping_logs table
+    op.create_table(
+        "scraping_logs",
+        sa.Column("id", sa.Integer(), primary_key=True, index=True),
+        sa.Column("date_execution", sa.DateTime(), nullable=True, index=True),
+        sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=True),
+        sa.Column("statut", sa.String(), nullable=False),
+        sa.Column("catalogues_trouves", sa.Integer(), default=0),
+        sa.Column("catalogues_nouveaux", sa.Integer(), default=0),
+        sa.Column("catalogues_mis_a_jour", sa.Integer(), default=0),
+        sa.Column("duree_secondes", sa.Float(), nullable=True),
+        sa.Column("message_erreur", sa.Text(), nullable=True),
+        sa.Column("details", sa.Text(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    """Drop catalog module tables."""
+    op.drop_table("scraping_logs")
+    op.drop_index("idx_catalogue_page_numero", table_name="catalogue_pages")
+    op.drop_table("catalogue_pages")
+    op.drop_table("catalogues")
+    op.drop_table("enseignes")

app/main.py

@@ -13,9 +13,10 @@ from sqlalchemy import text
 
 from app.database import SessionLocal, engine
 from app.limiter import limiter
-from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug
+from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues
 from app.services.scheduler_service import scheduled_refresh, scheduler
-from app.services import auth_service, search_service
+from app.services import auth_service, search_service, seed_enseignes
+from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler
 
 # Configure logging
 logging.basicConfig(
@@ -119,6 +120,13 @@ async def lifespan(app: FastAPI):
                 logger.info("Utilisateur admin par défaut créé (admin/admin)")
             else:
                 logger.info("Utilisateurs déjà initialisés")
+            
+            # Seed enseignes for catalog module
+            created_enseignes = seed_enseignes.seed_enseignes(db)
+            if created_enseignes > 0:
+                logger.info(f"{created_enseignes} enseigne(s) de catalogues créée(s)")
+            else:
+                 logger.info("Enseignes déjà initialisés")
         finally:
             db.close()
     except Exception as e:
@@ -127,10 +135,16 @@ async def lifespan(app: FastAPI):
     logger.info("Starting smart scheduler (Heartbeat: 1 minute)")
     scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True)
     scheduler.start()
+    
+    # Start catalog scraping scheduler
+    logger.info("Starting catalog scraping scheduler (6h and 18h daily)")
+    start_catalog_scheduler()
+    
     logger.info("Application started")
     yield
-    logger.info("Shutting down scheduler...")
+    logger.info("Shutting down schedulers...")
     scheduler.shutdown(wait=True)
+    stop_catalog_scheduler()
     logger.info("Application shutdown complete")
 
 
@@ -170,6 +184,9 @@ for router in [notifications.router, items.router, settings.router, jobs.router,
 app.include_router(search.router)
 app.include_router(search_sites.router)
 
+# Catalog router (already has /api prefix)
+app.include_router(catalogues.router)
+
 # Auth router (already has /api prefix)
 app.include_router(auth.router)
 

app/models.py

@@ -113,3 +113,87 @@ class NotificationChannel(Base):
     created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC))  # type: ignore
 
     items = relationship("Item", back_populates="notification_channel")
+
+
+# === CATALOG MODULE MODELS ===
+
+class Enseigne(Base):
+    """Stores/brands for promotional catalogs"""
+    __tablename__ = "enseignes"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    nom: str = Column(String, nullable=False)  # type: ignore  # "Gifi"
+    slug_bonial: str = Column(String, unique=True, nullable=False, index=True)  # type: ignore  # "Gifi"
+    logo_url: str | None = Column(String, nullable=True)  # type: ignore
+    couleur: str = Column(String, nullable=False)  # type: ignore  # "#E30613"
+    site_url: str | None = Column(String, nullable=True)  # type: ignore
+    description: str | None = Column(String, nullable=True)  # type: ignore
+    is_active: bool = Column(Boolean, default=True)  # type: ignore
+    ordre_affichage: int = Column(Integer, default=0)  # type: ignore
+    created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC))  # type: ignore
+    updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))  # type: ignore
+
+    # Relationships
+    catalogues = relationship("Catalogue", back_populates="enseigne", cascade="all, delete-orphan")
+    scraping_logs = relationship("ScrapingLog", back_populates="enseigne")
+
+
+class Catalogue(Base):
+    """Promotional catalogs from stores"""
+    __tablename__ = "catalogues"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    enseigne_id: int = Column(Integer, ForeignKey("enseignes.id"), nullable=False, index=True)  # type: ignore
+    titre: str = Column(String, nullable=False)  # type: ignore
+    description: str | None = Column(Text, nullable=True)  # type: ignore
+    date_debut: datetime = Column(DateTime, nullable=False, index=True)  # type: ignore
+    date_fin: datetime = Column(DateTime, nullable=False, index=True)  # type: ignore
+    image_couverture_url: str = Column(String, nullable=False)  # type: ignore
+    catalogue_url: str = Column(String, nullable=False)  # type: ignore  # URL Bonial viewer
+    nombre_pages: int = Column(Integer, default=0)  # type: ignore
+    statut: str = Column(String, default="actif", index=True)  # type: ignore  # actif, termine, erreur
+    content_hash: str = Column(String(64), unique=True, nullable=False, index=True)  # type: ignore  # SHA256
+    metadonnees: str | None = Column(Text, nullable=True)  # type: ignore  # JSON for additional data
+    created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC))  # type: ignore
+    updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))  # type: ignore
+
+    # Relationships
+    enseigne = relationship("Enseigne", back_populates="catalogues")
+    pages = relationship("CataloguePage", back_populates="catalogue", cascade="all, delete-orphan")
+
+
+class CataloguePage(Base):
+    """Individual pages of a catalog"""
+    __tablename__ = "catalogue_pages"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    catalogue_id: int = Column(Integer, ForeignKey("catalogues.id"), nullable=False, index=True)  # type: ignore
+    numero_page: int = Column(Integer, nullable=False)  # type: ignore
+    image_url: str = Column(String, nullable=False)  # type: ignore
+    image_thumbnail_url: str | None = Column(String, nullable=True)  # type: ignore
+    largeur: int | None = Column(Integer, nullable=True)  # type: ignore
+    hauteur: int | None = Column(Integer, nullable=True)  # type: ignore
+    created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC))  # type: ignore
+
+    # Relationship
+    catalogue = relationship("Catalogue", back_populates="pages")
+
+
+class ScrapingLog(Base):
+    """Logs of scraping executions"""
+    __tablename__ = "scraping_logs"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    date_execution: datetime = Column(DateTime, default=lambda: datetime.now(UTC), index=True)  # type: ignore
+    enseigne_id: int | None = Column(Integer, ForeignKey("enseignes.id"), nullable=True)  # type: ignore  # NULL = all stores
+    statut: str = Column(String, nullable=False)  # type: ignore  # success, error, partial
+    catalogues_trouves: int = Column(Integer, default=0)  # type: ignore
+    catalogues_nouveaux: int = Column(Integer, default=0)  # type: ignore
+    catalogues_mis_a_jour: int = Column(Integer, default=0)  # type: ignore
+    duree_secondes: float | None = Column(Float, nullable=True)  # type: ignore
+    message_erreur: str | None = Column(Text, nullable=True)  # type: ignore
+    details: str | None = Column(Text, nullable=True)  # type: ignore  # JSON
+
+    # Relationship
+    enseigne = relationship("Enseigne", back_populates="scraping_logs")
+

app/routers/catalogues.py

@@ -0,0 +1,317 @@
+"""
+API Router for catalog module
+Endpoints for enseignes, catalogues, and admin operations
+"""
+
+import logging
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+from sqlalchemy import func, and_
+
+from app.database import get_db
+from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
+from app.schemas_catalogues import (
+    EnseigneResponse,
+    CatalogueListResponse,
+    CatalogueDetailResponse,
+    CataloguePageResponse,
+    CataloguesPaginatedResponse,
+    PaginationMeta,
+    ScrapingLogResponse,
+    ScrapingStatsResponse,
+)
+from app.services.bonial_scraper import scrape_all_enseignes, scrape_enseigne
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/catalogues", tags=["catalogues"])
+
+
+# === Enseignes Endpoints ===
+
+@router.get("/enseignes", response_model=list[EnseigneResponse])
+async def get_enseignes(
+    db: Session = Depends(get_db),
+):
+    """Get all active enseignes with catalog counts"""
+    enseignes = (
+        db.query(Enseigne)
+        .filter(Enseigne.is_active == True)
+        .order_by(Enseigne.ordre_affichage)
+        .all()
+    )
+    
+    # Add catalog counts
+    result = []
+    for enseigne in enseignes:
+        count = (
+            db.query(func.count(Catalogue.id))
+            .filter(
+                and_(
+                    Catalogue.enseigne_id == enseigne.id,
+                    Catalogue.statut == "actif"
+                )
+            )
+            .scalar()
+        )
+        
+        enseigne_data = EnseigneResponse.model_validate(enseigne)
+        enseigne_data.catalogues_actifs_count = count
+        result.append(enseigne_data)
+    
+    return result
+
+
+# === Catalogues Endpoints ===
+
+@router.get("", response_model=CataloguesPaginatedResponse)
+async def get_catalogues(
+    enseigne_ids: str | None = Query(None, description="Comma-separated enseigne IDs"),
+    statut: str | None = Query(None, description="Filter by status: actif, termine, erreur, tous"),
+    date_debut_min: datetime | None = Query(None, description="Start date minimum"),
+    date_fin_max: datetime | None = Query(None, description="End date maximum"),
+    recherche: str | None = Query(None, description="Search in titles"),
+    page: int = Query(1, ge=1, description="Page number"),
+    limit: int = Query(20, ge=1, le=100, description="Items per page"),
+    sort: str = Query("date_debut", description="Sort by: date_debut, date_fin, enseigne"),
+    order: str = Query("desc", description="Order: asc, desc"),
+    db: Session = Depends(get_db),
+):
+    """Get catalogues with filters and pagination"""
+    
+    # Build query
+    query = db.query(Catalogue)
+    
+    # Filter by enseignes
+    if enseigne_ids:
+        try:
+            ids = [int(id_str.strip()) for id_str in enseigne_ids.split(",")]
+            query = query.filter(Catalogue.enseigne_id.in_(ids))
+        except ValueError:
+            raise HTTPException(status_code=400, detail="Invalid enseigne_ids format")
+    
+    # Filter by status
+    if statut and statut != "tous":
+        query = query.filter(Catalogue.statut == statut)
+    
+    # Filter by dates
+    if date_debut_min:
+        query = query.filter(Catalogue.date_debut >= date_debut_min)
+    if date_fin_max:
+        query = query.filter(Catalogue.date_fin <= date_fin_max)
+    
+    # Search in titles
+    if recherche:
+        query = query.filter(Catalogue.titre.ilike(f"%{recherche}%"))
+    
+    # Sort
+    sort_column = {
+        "date_debut": Catalogue.date_debut,
+        "date_fin": Catalogue.date_fin,
+        "enseigne": Catalogue.enseigne_id,
+    }.get(sort, Catalogue.date_debut)
+    
+    if order == "desc":
+        query = query.order_by(sort_column.desc())
+    else:
+        query = query.order_by(sort_column.asc())
+    
+    # Count total
+    total = query.count()
+    
+    # Paginate
+    offset = (page - 1) * limit
+    catalogues = query.offset(offset).limit(limit).all()
+    
+    # Build response
+    data = []
+    for catalogue in catalogues:
+        cat_response = CatalogueListResponse.model_validate(catalogue)
+        data.append(cat_response)
+    
+    pagination = PaginationMeta(
+        total=total,
+        page=page,
+        limit=limit,
+        pages_total=(total + limit - 1) // limit,
+    )
+    
+    # Get last scraping time
+    last_scraping = (
+        db.query(ScrapingLog)
+        .order_by(ScrapingLog.date_execution.desc())
+        .first()
+    )
+    
+    metadata = {
+        "derniere_mise_a_jour": last_scraping.date_execution if last_scraping else None
+    }
+    
+    return CataloguesPaginatedResponse(
+        data=data,
+        pagination=pagination,
+        metadata=metadata,
+    )
+
+
+@router.get("/{catalogue_id}", response_model=CatalogueDetailResponse)
+async def get_catalogue_detail(
+    catalogue_id: int,
+    db: Session = Depends(get_db),
+):
+    """Get detailed information for a specific catalogue"""
+    catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
+    
+    if not catalogue:
+        raise HTTPException(status_code=404, detail="Catalogue not found")
+    
+    return CatalogueDetailResponse.model_validate(catalogue)
+
+
+@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse])
+async def get_catalogue_pages(
+    catalogue_id: int,
+    db: Session = Depends(get_db),
+):
+    """Get all pages of a catalogue"""
+    # Verify catalogue exists
+    catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
+    
+    if not catalogue:
+        raise HTTPException(status_code=404, detail="Catalogue not found")
+    
+    # Get pages ordered by page number
+    pages = (
+        db.query(CataloguePage)
+        .filter(CataloguePage.catalogue_id == catalogue_id)
+        .order_by(CataloguePage.numero_page)
+        .all()
+    )
+    
+    return [CataloguePageResponse.model_validate(page) for page in pages]
+
+
+# === Admin Endpoints ===
+
+@router.post("/admin/scraping/trigger", response_model=dict)
+async def trigger_scraping(
+    enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"),
+    db: Session = Depends(get_db),
+):
+    """Manually trigger catalog scraping"""
+    
+    if enseigne_id:
+        # Scrape single enseigne
+        enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first()
+        
+        if not enseigne:
+            raise HTTPException(status_code=404, detail="Enseigne not found")
+        
+        # Run scraping asynchronously (in background)
+        import asyncio
+        from app.services.bonial_scraper import scrape_enseigne as scrape_ens
+        
+        log = await scrape_ens(enseigne, db)
+        
+        return {
+            "success": True,
+            "message": f"Scraping complete for {enseigne.nom}",
+            "catalogues_trouves": log.catalogues_trouves,
+            "catalogues_nouveaux": log.catalogues_nouveaux,
+        }
+    else:
+        # Scrape all enseignes
+        logs = await scrape_all_enseignes(db)
+        
+        total_new = sum(log.catalogues_nouveaux for log in logs)
+        total_found = sum(log.catalogues_trouves for log in logs)
+        
+        return {
+            "success": True,
+            "message": f"Scraping complete for all enseignes",
+            "enseignes_processed": len(logs),
+            "catalogues_trouves": total_found,
+            "catalogues_nouveaux": total_new,
+        }
+
+
+@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse])
+async def get_scraping_logs(
+    enseigne_id: int | None = Query(None, description="Filter by enseigne"),
+    statut: str | None = Query(None, description="Filter by status"),
+    limit: int = Query(50, ge=1, le=200, description="Number of logs to return"),
+    db: Session = Depends(get_db),
+):
+    """Get scraping execution logs"""
+    query = db.query(ScrapingLog)
+    
+    if enseigne_id:
+        query = query.filter(ScrapingLog.enseigne_id == enseigne_id)
+    
+    if statut:
+        query = query.filter(ScrapingLog.statut == statut)
+    
+    logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all()
+    
+    # Add enseigne name
+    result = []
+    for log in logs:
+        log_dict = ScrapingLogResponse.model_validate(log).model_dump()
+        if log.enseigne_id:
+            enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first()
+            log_dict["enseigne_nom"] = enseigne.nom if enseigne else None
+        else:
+            log_dict["enseigne_nom"] = "Toutes"
+        
+        result.append(ScrapingLogResponse(**log_dict))
+    
+    return result
+
+
+@router.get("/admin/stats", response_model=ScrapingStatsResponse)
+async def get_scraping_stats(
+    db: Session = Depends(get_db),
+):
+    """Get scraping statistics"""
+    
+    # Total catalogues
+    total_catalogues = db.query(func.count(Catalogue.id)).scalar()
+    
+    # Catalogues par enseigne
+    catalogues_par_enseigne = {}
+    enseignes = db.query(Enseigne).all()
+    
+    for enseigne in enseignes:
+        count = (
+            db.query(func.count(Catalogue.id))
+            .filter(Catalogue.enseigne_id == enseigne.id)
+            .scalar()
+        )
+        catalogues_par_enseigne[enseigne.nom] = count
+    
+    # Last scraping time
+    last_log = (
+        db.query(ScrapingLog)
+        .order_by(ScrapingLog.date_execution.desc())
+        .first()
+    )
+    
+    derniere_mise_a_jour = last_log.date_execution if last_log else None
+    
+    # Next execution (6:00 or 18:00)
+    now = datetime.now()
+    if now.hour < 6:
+        prochaine = f"Aujourd'hui à 6:00"
+    elif now.hour < 18:
+        prochaine = f"Aujourd'hui à 18:00"
+    else:
+        prochaine = f"Demain à 6:00"
+    
+    return ScrapingStatsResponse(
+        total_catalogues=total_catalogues,
+        catalogues_par_enseigne=catalogues_par_enseigne,
+        derniere_mise_a_jour=derniere_mise_a_jour,
+        prochaine_execution=prochaine,
+    )

app/schemas_catalogues.py

@@ -0,0 +1,109 @@
+"""
+Pydantic schemas for catalog module API
+"""
+
+from datetime import datetime
+from pydantic import BaseModel
+
+
+# === Enseigne Schemas ===
+
+class EnseigneResponse(BaseModel):
+    id: int
+    nom: str
+    slug_bonial: str
+    logo_url: str | None
+    couleur: str
+    site_url: str | None
+    description: str | None
+    is_active: bool
+    ordre_affichage: int
+    
+    # Statistics
+    catalogues_actifs_count: int | None = None
+    
+    model_config = {"from_attributes": True}
+
+
+# === Catalogue Schemas ===
+
+class CatalogueListResponse(BaseModel):
+    id: int
+    enseigne: EnseigneResponse
+    titre: str
+    date_debut: datetime
+    date_fin: datetime
+    image_couverture_url: str
+    statut: str
+    nombre_pages: int
+    created_at: datetime
+    
+    model_config = {"from_attributes": True}
+
+
+class CatalogueDetailResponse(BaseModel):
+    id: int
+    enseigne: EnseigneResponse
+    titre: str
+    description: str | None
+    date_debut: datetime
+    date_fin: datetime
+    image_couverture_url: str
+    catalogue_url: str
+    statut: str
+    nombre_pages: int
+    metadonnees: str | None
+    created_at: datetime
+    updated_at: datetime
+    
+    model_config = {"from_attributes": True}
+
+
+class CataloguePageResponse(BaseModel):
+    id: int
+    numero_page: int
+    image_url: str
+    image_thumbnail_url: str | None
+    largeur: int | None
+    hauteur: int | None
+    
+    model_config = {"from_attributes": True}
+
+
+# === Pagination ===
+
+class PaginationMeta(BaseModel):
+    total: int
+    page: int
+    limit: int
+    pages_total: int
+
+
+class CataloguesPaginatedResponse(BaseModel):
+    data: list[CatalogueListResponse]
+    pagination: PaginationMeta
+    metadata: dict
+
+
+# === Admin Schemas ===
+
+class ScrapingLogResponse(BaseModel):
+    id: int
+    date_execution: datetime
+    enseigne_id: int | None
+    enseigne_nom: str | None
+    statut: str
+    catalogues_trouves: int
+    catalogues_nouveaux: int
+    catalogues_mis_a_jour: int
+    duree_secondes: float | None
+    message_erreur: str | None
+    
+    model_config = {"from_attributes": True}
+
+
+class ScrapingStatsResponse(BaseModel):
+    total_catalogues: int
+    catalogues_par_enseigne: dict[str, int]
+    derniere_mise_a_jour: datetime | None
+    prochaine_execution: str | None

app/services/bonial_scraper.py

@@ -0,0 +1,400 @@
+"""
+Bonial.fr Scraper Service
+
+Scrapes promotional catalogs from Bonial.fr for configured enseignes.
+Uses Playwright for browser automation.
+"""
+
+import asyncio
+import hashlib
+import logging
+import re
+from datetime import datetime
+from typing import Any
+
+from playwright.async_api import async_playwright, Page, Browser
+from sqlalchemy.orm import Session
+
+from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
+
+logger = logging.getLogger(__name__)
+
+# Bonial selectors based on manual analysis
+BONIAL_SELECTORS = {
+    "catalog_card": "section:has(img[src*='content-media.bonial.biz'])",
+    "catalog_title": "div.text-dark.text-lg.font-bold, h3, h2",
+    "catalog_image": "img[src*='content-media.bonial.biz']",
+    "catalog_link": "a:has-text('Ouvrir le catalogue')",
+    "cookie_accept": "button:has-text('OK'), button:has-text('Accepter')",
+}
+
+# Date pattern: "mar. 25/11 - lun. 08/12/2025"
+DATE_PATTERN = r"(\d{2}/\d{2})\s*-\s*(\d{2}/\d{2}/\d{4})"
+
+
+def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
+    """
+    Parse Bonial date format to datetime objects.
+    
+    Args:
+        date_str: Date string like "mar. 25/11 - lun. 08/12/2025"
+    
+    Returns:
+        Tuple of (date_debut, date_fin)
+    
+    Raises:
+        ValueError: If date format is invalid
+    """
+    match = re.search(DATE_PATTERN, date_str)
+    
+    if not match:
+        raise ValueError(f"Invalid date format: {date_str}")
+    
+    debut_str, fin_str = match.groups()
+    
+    # Extract year from end date
+    year = fin_str.split("/")[2]
+    
+    # Add year to start date
+    debut_full = f"{debut_str}/{year}"
+    
+    # Parse dates
+    debut = datetime.strptime(debut_full, "%d/%m/%Y")
+    fin = datetime.strptime(fin_str, "%d/%m/%Y")
+    
+    return debut, fin
+
+
+def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str:
+    """
+    Compute SHA256 hash for duplicate detection.
+    
+    Args:
+        enseigne_id: ID of the enseigne
+        titre: Catalog title
+        date_debut: Start date
+    
+    Returns:
+        SHA256 hash string
+    """
+    content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}"
+    return hashlib.sha256(content.encode()).hexdigest()
+
+
+async def accept_cookies(page: Page) -> None:
+    """Accept cookie consent banner if present."""
+    try:
+        await page.click(BONIAL_SELECTORS["cookie_accept"], timeout=3000)
+        logger.debug("Accepted cookies")
+        await asyncio.sleep(1)
+    except Exception:
+        logger.debug("No cookie banner found or already accepted")
+
+
+async def scrape_catalog_list(
+    page: Page,
+    enseigne: Enseigne
+) -> list[dict[str, Any]]:
+    """
+    Scrape the list of catalogs for an ens enseigne from Bonial.
+    
+    Args:
+        page: Playwright page object
+        enseigne: Enseigne object
+    
+    Returns:
+        List of catalog data dictionaries
+    """
+    url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}"
+    logger.info(f"Scraping catalog list for {enseigne.nom} from {url}")
+    
+    await page.goto(url, wait_until="networkidle")
+    await accept_cookies(page)
+    
+    # Wait for catalog cards to load
+    try:
+        await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000)
+    except Exception as e:
+        logger.error(f"No catalog cards found for {enseigne.nom}: {e}")
+        return []
+    
+    # Extract all catalog cards
+    cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"])
+    logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}")
+    
+    catalogues = []
+    
+    for idx, card in enumerate(cards):
+        try:
+            # Extract title
+            title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"])
+            titre = await title_el.inner_text() if title_el else None
+            
+            # Extract dates - look for any element containing date pattern
+            dates_text = await card.inner_text()
+            date_match = re.search(DATE_PATTERN, dates_text)
+            
+            # Extract image
+            image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"])
+            image_url = await image_el.get_attribute("src") if image_el else None
+            
+            # Extract link - try multiple strategies
+            link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"])
+            if not link_el:
+                # Fallback: click on image
+                link_el = image_el
+            
+            catalogue_url = await link_el.get_attribute("href") if link_el else None
+            
+            # Make URL absolute if relative
+            if catalogue_url and not catalogue_url.startswith("http"):
+                catalogue_url = f"https://www.bonial.fr{catalogue_url}"
+            
+            if all([titre, date_match, image_url, catalogue_url]):
+                # Parse dates
+                try:
+                    date_debut, date_fin = parse_bonial_dates(dates_text)
+                except ValueError as e:
+                    logger.warning(f"Failed to parse dates for catalog {idx}: {e}")
+                    continue
+                
+                catalogues.append({
+                    "titre": titre.strip(),
+                    "date_debut": date_debut,
+                    "date_fin": date_fin,
+                    "image_couverture_url": image_url,
+                    "catalogue_url": catalogue_url,
+                })
+                
+                logger.debug(f"Extracted catalog: {titre}")
+            else:
+                logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={bool(date_match)}, image={bool(image_url)}, url={bool(catalogue_url)}")
+        
+        except Exception as e:
+            logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}")
+            continue
+    
+    logger.info(f"Successfully extracted {len(catalogues)} catalogs for {enseigne.nom}")
+    return catalogues
+
+
+async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]:
+    """
+    Scrape all pages of a catalog from the Bonial viewer.
+    
+    Args:
+        page: Playwright page object
+        catalogue_url: URL of the catalog viewer
+    
+    Returns:
+        List of page data dictionaries
+    """
+    logger.info(f"Scraping catalog pages from {catalogue_url}")
+    
+    await page.goto(catalogue_url, wait_until="networkidle")
+    await asyncio.sleep(2)  # Wait for viewer to initialize
+    
+    # Try to close any popup/modal
+    try:
+        await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000)
+        await asyncio.sleep(1)
+    except Exception:
+        pass
+    
+    # Find all images from content-media.bonial.biz
+    images = await page.query_selector_all("img[src*='content-media.bonial.biz']")
+    
+    if not images:
+        logger.warning(f"No images found in catalog viewer: {catalogue_url}")
+        return []
+    
+    # Extract unique image URLs (filter duplicates and thumbnails)
+    seen_urls = set()
+    pages = []
+    
+    for idx, img in enumerate(images):
+        try:
+            src = await img.get_attribute("src")
+            
+            if src and src not in seen_urls:
+                # Filter out blurred/low-res images if possible
+                # Full res images are typically larger
+                width = await img.evaluate("el => el.naturalWidth")
+                height = await img.evaluate("el => el.naturalHeight")
+                
+                # Skip very small images (likely thumbnails)
+                if width < 200 or height < 200:
+                    continue
+                
+                seen_urls.add(src)
+                pages.append({
+                    "numero_page": len(pages) + 1,
+                    "image_url": src,
+                    "largeur": width,
+                    "hauteur": height,
+                })
+        
+        except Exception as e:
+            logger.debug(f"Error processing image {idx}: {e}")
+            continue
+    
+    logger.info(f"Found {len(pages)} pages in catalog")
+    return pages
+
+
+async def scrape_enseigne(
+    enseigne: Enseigne,
+    db: Session,
+    browser: Browser | None = None
+) -> ScrapingLog:
+    """
+    Scrape all catalogs for a specific enseigne.
+    
+    Args:
+        enseigne: Enseigne to scrape
+        db: Database session
+        browser: Optional existing browser instance
+    
+    Returns:
+        ScrapingLog object with execution details
+    """
+    start_time = datetime.now()
+    log = ScrapingLog(
+        enseigne_id=enseigne.id,
+        statut="error",  # Will be updated
+        catalogues_trouves=0,
+        catalogues_nouveaux=0,
+        catalogues_mis_a_jour=0,
+    )
+    
+    own_browser = browser is None
+    
+    try:
+        # Create browser if not provided
+        if own_browser:
+            p = await async_playwright().start()
+            browser = await p.chromium.launch(headless=True)
+        
+        page = await browser.new_page()
+        
+        # Set realistic user agent
+        await page.set_extra_http_headers({
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+        })
+        
+        # Scrape catalog list
+        catalogues_data = await scrape_catalog_list(page, enseigne)
+        log.catalogues_trouves = len(catalogues_data)
+        
+        if not catalogues_data:
+            log.statut = "success"
+            log.message_erreur = "No catalogs found (may be normal)"
+            await page.close()
+            return log
+        
+        # Process each catalog
+        for cat_data in catalogues_data:
+            try:
+                # Compute hash for duplicate detection
+                content_hash = compute_catalog_hash(
+                    enseigne.id,
+                    cat_data["titre"],
+                    cat_data["date_debut"]
+                )
+                
+                # Check if catalog already exists
+                existing = db.query(Catalogue).filter_by(content_hash=content_hash).first()
+                
+                if existing:
+                    logger.debug(f"Catalog already exists: {cat_data['titre']}")
+                    continue
+                
+                # Scrape catalog pages
+                await asyncio.sleep(3)  # Respectful delay
+                pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"])
+                
+                # Create new catalog
+                catalogue = Catalogue(
+                    enseigne_id=enseigne.id,
+                    titre=cat_data["titre"],
+                    date_debut=cat_data["date_debut"],
+                    date_fin=cat_data["date_fin"],
+                    image_couverture_url=cat_data["image_couverture_url"],
+                    catalogue_url=cat_data["catalogue_url"],
+                    nombre_pages=len(pages_data),
+                    content_hash=content_hash,
+                    statut="actif" if cat_data["date_fin"] >= datetime.now() else "termine",
+                )
+                
+                db.add(catalogue)
+                db.flush()  # Get catalogue.id
+                
+                # Add pages
+                for page_data in pages_data:
+                    cat_page = CataloguePage(
+                        catalogue_id=catalogue.id,
+                        **page_data
+                    )
+                    db.add(cat_page)
+                
+                log.catalogues_nouveaux += 1
+                logger.info(f"Created catalog: {cat_data['titre']} ({len(pages_data)} pages)")
+            
+            except Exception as e:
+                logger.error(f"Error processing catalog {cat_data.get('titre')}: {e}")
+                continue
+        
+        await page.close()
+        
+        # Update status
+        log.statut = "success" if log.catalogues_nouveaux > 0 else "partial"
+        db.commit()
+    
+    except Exception as e:
+        logger.error(f"Error scraping enseigne {enseigne.nom}: {e}")
+        log.statut = "error"
+        log.message_erreur = str(e)
+        db.rollback()
+    
+    finally:
+        if own_browser and browser:
+            await browser.close()
+        
+        # Record duration
+        log.duree_secondes = (datetime.now() - start_time).total_seconds()
+        db.add(log)
+        db.commit()
+    
+    return log
+
+
+async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
+    """
+    Scrape all active enseignes.
+    
+    Args:
+        db: Database session
+    
+    Returns:
+        List of ScrapingLog objects
+    """
+    enseignes = db.query(Enseigne).filter_by(is_active=True).all()
+    logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
+    
+    logs = []
+    
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        
+        for enseigne in enseignes:
+            try:
+                await asyncio.sleep(2)  # Delay between enseignes
+                log = await scrape_enseigne(enseigne, db, browser)
+                logs.append(log)
+            except Exception as e:
+                logger.error(f"Failed to scrape {enseigne.nom}: {e}")
+        
+        await browser.close()
+    
+    logger.info(f"Scraping complete: {len(logs)} enseignes processed")
+    return logs

app/services/scheduler.py

@@ -0,0 +1,62 @@
+"""
+Scheduler for automated catalog scraping
+
+Runs scraping jobs twice daily (6h and 18h) for all active enseignes.
+"""
+
+import logging
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.cron import CronTrigger
+
+from app.database import SessionLocal
+from app.services.bonial_scraper import scrape_all_enseignes
+
+logger = logging.getLogger(__name__)
+
+# Global scheduler instance
+scheduler = AsyncIOScheduler()
+
+
+async def scheduled_scrape_job():
+    """Scheduled job to scrape all active enseignes"""
+    logger.info("Starting scheduled catalog scraping job")
+    
+    db = SessionLocal()
+    try:
+        logs = await scrape_all_ense ignes(db)
+        
+        total_new = sum(log.catalogues_nouveaux for log in logs)
+        total_found = sum(log.catalogues_trouves for log in logs)
+        
+        logger.info(
+            f"Scheduled scraping complete: "
+            f"{len(logs)} enseignes processed, "
+            f"{total_found} catalogues found, "
+            f"{total_new} new catalogues created"
+        )
+    except Exception as e:
+        logger.error(f"Error in scheduled scraping job: {e}")
+    finally:
+        db.close()
+
+
+def start_scheduler():
+    """Start the scheduler with configured jobs"""
+    # Schedule scraping twice daily at 6:00 and 18:00
+    scheduler.add_job(
+        scheduled_scrape_job,
+        CronTrigger(hour="6,18", minute=0),
+        id="catalog_scraping",
+        name="Scrape Bonial catalogs",
+        replace_existing=True,
+    )
+    
+    scheduler.start()
+    logger.info("Catalog scraping scheduler started (runs at 6:00 and 18:00)")
+
+
+def stop_scheduler():
+    """Stop the scheduler"""
+    if scheduler.running:
+        scheduler.shutdown()
+        logger.info("Catalog scraping scheduler stopped")

app/services/seed_enseignes.py

@@ -0,0 +1,128 @@
+"""
+Seed default enseignes (stores) for the catalog module
+"""
+
+import logging
+from sqlalchemy.orm import Session
+
+from app.models import Enseigne
+
+logger = logging.getLogger(__name__)
+
+# Data for the 9 enseignes
+ENSEIGNES_DATA = [
+    {
+        "nom": "Gifi",
+        "slug_bonial": "Gifi",
+        "couleur": "#E30613",
+        "site_url": "https://www.gifi.fr",
+        "description": "Décoration, maison, bazar",
+        "ordre_affichage": 1,
+    },
+    {
+        "nom": "Action",
+        "slug_bonial": "Action",
+        "couleur": "#0066B3",
+        "site_url": "https://www.action.com/fr-fr/",
+        "description": "Discount non-alimentaire",
+        "ordre_affichage": 2,
+    },
+    {
+        "nom": "Centrakor",
+        "slug_bonial": "Centrakor",
+        "couleur": "#E94E1B",
+        "site_url": "https://www.centrakor.com",
+        "description": "Décoration, maison",
+        "ordre_affichage": 3,
+    },
+    {
+        "nom": "La Foir'Fouille",
+        "slug_bonial": "La-Foir-Fouille",
+        "couleur": "#009639",
+        "site_url": "https://www.lafoirfouille.fr",
+        "description": "Bazar, décoration",
+        "ordre_affichage": 4,
+    },
+    {
+        "nom": "Stokomani",
+        "slug_bonial": "Stokomani",
+        "couleur": "#FF6600",
+        "site_url": "https://www.stokomani.fr",
+        "description": "Déstockage textile et maison",
+        "ordre_affichage": 5,
+    },
+    {
+        "nom": "B&M",
+        "slug_bonial": "BM",
+        "couleur": "#D4145A",
+        "site_url": "https://bmstores.fr",
+        "description": "Discount britannique",
+        "ordre_affichage": 6,
+    },
+    {
+        "nom": "L'Incroyable",
+        "slug_bonial": "L-incroyable",
+        "couleur": "#8B0000",
+        "site_url": "https://www.lincroyable.fr",
+        "description": "Décoration et mobilier discount (Groupe Althys, siège à Denain)",
+        "ordre_affichage": 7,
+    },
+    {
+        "nom": "Bazarland",
+        "slug_bonial": "Bazarland",
+        "couleur": "#FFCC00",
+        "site_url": None,
+        "description": "Bazar discount",
+        "ordre_affichage": 8,
+    },
+    {
+        "nom": "Noz",
+        "slug_bonial": "Noz",
+        "couleur": "#003366",
+        "site_url": "https://www.noz.fr",
+        "description": "Déstockage généraliste",
+        "ordre_affichage": 9,
+    },
+]
+
+
+def seed_enseignes(db: Session) -> int:
+    """
+    Seed the default enseignes if they don't exist.
+    
+    Returns:
+        Number of enseignes created
+    """
+    created = 0
+    
+    for data in ENSEIGNES_DATA:
+        # Check if enseigne already exists
+        existing = db.query(Enseigne).filter_by(slug_bonial=data["slug_bonial"]).first()
+        
+        if not existing:
+            enseigne = Enseigne(**data)
+            db.add(enseigne)
+            created += 1
+            logger.info(f"Created enseigne: {data['nom']}")
+        else:
+            logger.debug(f"Enseigne already exists: {data['nom']}")
+    
+    db.commit()
+    logger.info(f"Seeding complete: {created} enseigne(s) created")
+    
+    return created
+
+
+def get_enseigne_by_slug(db: Session, slug: str) -> Enseigne | None:
+    """Get an enseigne by its Bonial slug"""
+    return db.query(Enseigne).filter_by(slug_bonial=slug).first()
+
+
+def get_all_active_enseignes(db: Session) -> list[Enseigne]:
+    """Get all active enseignes ordered by display order"""
+    return (
+        db.query(Enseigne)
+        .filter_by(is_active=True)
+        .order_by(Enseigne.ordre_affichage)
+        .all()
+    )