diff --git a/README.md b/README.md index 18bd25a..6f95cfd 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ - **Historique visuel** : Conserve l'historique des captures d'écran avec les métadonnées IA - **Défilement intelligent** : Scroll automatique pour charger le contenu différé - **Notifications** : Support multi-canaux (Discord, Telegram, Email, etc.) via [Apprise](https://github.com/caronc/apprise) +- **Catalogues promotionnels** : Consultation des catalogues et prospectus discount (Gifi, Action, Centrakor, etc.) scrapés depuis Bonial.fr - **Interface en français** : Application entièrement traduite en français -- **Mode sombre** : Interface moderne avec support du thème clair/sombre +- **Mode sombre** : Interface moderne avec support du thème clair/foncé - **Dockerisé** : Déploiement facile avec Docker Compose ## Prérequis diff --git a/app/alembic/versions/c1d2e3f4g5h6_add_catalog_module.py b/app/alembic/versions/c1d2e3f4g5h6_add_catalog_module.py new file mode 100644 index 0000000..6547de6 --- /dev/null +++ b/app/alembic/versions/c1d2e3f4g5h6_add_catalog_module.py @@ -0,0 +1,103 @@ +"""Add catalog module tables + +Revision ID: c1d2e3f4g5h6 +Revises: b2c3d4e5f6g7 +Create Date: 2025-11-29 18:56:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c1d2e3f4g5h6" +down_revision: str | None = "b2c3d4e5f6g7" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Create tables for the catalog module.""" + + # Create enseignes table + op.create_table( + "enseignes", + sa.Column("id", sa.Integer(), primary_key=True, index=True), + sa.Column("nom", sa.String(), nullable=False), + sa.Column("slug_bonial", sa.String(), nullable=False, unique=True, index=True), + sa.Column("logo_url", sa.String(), nullable=True), + sa.Column("couleur", sa.String(), nullable=False), + sa.Column("site_url", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("is_active", sa.Boolean(), default=True), + sa.Column("ordre_affichage", sa.Integer(), default=0), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("updated_at", sa.DateTime(), nullable=True), + ) + + # Create catalogues table + op.create_table( + "catalogues", + sa.Column("id", sa.Integer(), primary_key=True, index=True), + sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=False, index=True), + sa.Column("titre", sa.String(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("date_debut", sa.DateTime(), nullable=False, index=True), + sa.Column("date_fin", sa.DateTime(), nullable=False, index=True), + sa.Column("image_couverture_url", sa.String(), nullable=False), + sa.Column("catalogue_url", sa.String(), nullable=False), + sa.Column("nombre_pages", sa.Integer(), default=0), + sa.Column("statut", sa.String(), default="actif", index=True), + sa.Column("content_hash", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("metadonnees", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("updated_at", sa.DateTime(), nullable=True), + ) + + # Create catalogue_pages table + op.create_table( + "catalogue_pages", + sa.Column("id", sa.Integer(), primary_key=True, index=True), + sa.Column("catalogue_id", sa.Integer(), sa.ForeignKey("catalogues.id"), nullable=False, index=True), + sa.Column("numero_page", sa.Integer(), nullable=False), + sa.Column("image_url", sa.String(), nullable=False), + sa.Column("image_thumbnail_url", sa.String(), nullable=True), + sa.Column("largeur", sa.Integer(), nullable=True), + sa.Column("hauteur", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + ) + + # Create unique index on catalogue_id + numero_page + op.create_index( + "idx_catalogue_page_numero", + "catalogue_pages", + ["catalogue_id", "numero_page"], + unique=True, + ) + + # Create scraping_logs table + op.create_table( + "scraping_logs", + sa.Column("id", sa.Integer(), primary_key=True, index=True), + sa.Column("date_execution", sa.DateTime(), nullable=True, index=True), + sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=True), + sa.Column("statut", sa.String(), nullable=False), + sa.Column("catalogues_trouves", sa.Integer(), default=0), + sa.Column("catalogues_nouveaux", sa.Integer(), default=0), + sa.Column("catalogues_mis_a_jour", sa.Integer(), default=0), + sa.Column("duree_secondes", sa.Float(), nullable=True), + sa.Column("message_erreur", sa.Text(), nullable=True), + sa.Column("details", sa.Text(), nullable=True), + ) + + +def downgrade() -> None: + """Drop catalog module tables.""" + op.drop_table("scraping_logs") + op.drop_index("idx_catalogue_page_numero", table_name="catalogue_pages") + op.drop_table("catalogue_pages") + op.drop_table("catalogues") + op.drop_table("enseignes") diff --git a/app/main.py b/app/main.py index a760996..954f636 100644 --- a/app/main.py +++ b/app/main.py @@ -13,9 +13,10 @@ from sqlalchemy import text from app.database import SessionLocal, engine from app.limiter import limiter -from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug +from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues from app.services.scheduler_service import scheduled_refresh, scheduler -from app.services import auth_service, search_service +from app.services import auth_service, search_service, seed_enseignes +from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler # Configure logging logging.basicConfig( @@ -119,6 +120,13 @@ async def lifespan(app: FastAPI): logger.info("Utilisateur admin par défaut créé (admin/admin)") else: logger.info("Utilisateurs déjà initialisés") + + # Seed enseignes for catalog module + created_enseignes = seed_enseignes.seed_enseignes(db) + if created_enseignes > 0: + logger.info(f"{created_enseignes} enseigne(s) de catalogues créée(s)") + else: + logger.info("Enseignes déjà initialisés") finally: db.close() except Exception as e: @@ -127,10 +135,16 @@ async def lifespan(app: FastAPI): logger.info("Starting smart scheduler (Heartbeat: 1 minute)") scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True) scheduler.start() + + # Start catalog scraping scheduler + logger.info("Starting catalog scraping scheduler (6h and 18h daily)") + start_catalog_scheduler() + logger.info("Application started") yield - logger.info("Shutting down scheduler...") + logger.info("Shutting down schedulers...") scheduler.shutdown(wait=True) + stop_catalog_scheduler() logger.info("Application shutdown complete") @@ -170,6 +184,9 @@ for router in [notifications.router, items.router, settings.router, jobs.router, app.include_router(search.router) app.include_router(search_sites.router) +# Catalog router (already has /api prefix) +app.include_router(catalogues.router) + # Auth router (already has /api prefix) app.include_router(auth.router) diff --git a/app/models.py b/app/models.py index d5d61e3..cad1e83 100644 --- a/app/models.py +++ b/app/models.py @@ -113,3 +113,87 @@ class NotificationChannel(Base): created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore items = relationship("Item", back_populates="notification_channel") + + +# === CATALOG MODULE MODELS === + +class Enseigne(Base): + """Stores/brands for promotional catalogs""" + __tablename__ = "enseignes" + + id: int = Column(Integer, primary_key=True, index=True) # type: ignore + nom: str = Column(String, nullable=False) # type: ignore # "Gifi" + slug_bonial: str = Column(String, unique=True, nullable=False, index=True) # type: ignore # "Gifi" + logo_url: str | None = Column(String, nullable=True) # type: ignore + couleur: str = Column(String, nullable=False) # type: ignore # "#E30613" + site_url: str | None = Column(String, nullable=True) # type: ignore + description: str | None = Column(String, nullable=True) # type: ignore + is_active: bool = Column(Boolean, default=True) # type: ignore + ordre_affichage: int = Column(Integer, default=0) # type: ignore + created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore + updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore + + # Relationships + catalogues = relationship("Catalogue", back_populates="enseigne", cascade="all, delete-orphan") + scraping_logs = relationship("ScrapingLog", back_populates="enseigne") + + +class Catalogue(Base): + """Promotional catalogs from stores""" + __tablename__ = "catalogues" + + id: int = Column(Integer, primary_key=True, index=True) # type: ignore + enseigne_id: int = Column(Integer, ForeignKey("enseignes.id"), nullable=False, index=True) # type: ignore + titre: str = Column(String, nullable=False) # type: ignore + description: str | None = Column(Text, nullable=True) # type: ignore + date_debut: datetime = Column(DateTime, nullable=False, index=True) # type: ignore + date_fin: datetime = Column(DateTime, nullable=False, index=True) # type: ignore + image_couverture_url: str = Column(String, nullable=False) # type: ignore + catalogue_url: str = Column(String, nullable=False) # type: ignore # URL Bonial viewer + nombre_pages: int = Column(Integer, default=0) # type: ignore + statut: str = Column(String, default="actif", index=True) # type: ignore # actif, termine, erreur + content_hash: str = Column(String(64), unique=True, nullable=False, index=True) # type: ignore # SHA256 + metadonnees: str | None = Column(Text, nullable=True) # type: ignore # JSON for additional data + created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore + updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore + + # Relationships + enseigne = relationship("Enseigne", back_populates="catalogues") + pages = relationship("CataloguePage", back_populates="catalogue", cascade="all, delete-orphan") + + +class CataloguePage(Base): + """Individual pages of a catalog""" + __tablename__ = "catalogue_pages" + + id: int = Column(Integer, primary_key=True, index=True) # type: ignore + catalogue_id: int = Column(Integer, ForeignKey("catalogues.id"), nullable=False, index=True) # type: ignore + numero_page: int = Column(Integer, nullable=False) # type: ignore + image_url: str = Column(String, nullable=False) # type: ignore + image_thumbnail_url: str | None = Column(String, nullable=True) # type: ignore + largeur: int | None = Column(Integer, nullable=True) # type: ignore + hauteur: int | None = Column(Integer, nullable=True) # type: ignore + created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore + + # Relationship + catalogue = relationship("Catalogue", back_populates="pages") + + +class ScrapingLog(Base): + """Logs of scraping executions""" + __tablename__ = "scraping_logs" + + id: int = Column(Integer, primary_key=True, index=True) # type: ignore + date_execution: datetime = Column(DateTime, default=lambda: datetime.now(UTC), index=True) # type: ignore + enseigne_id: int | None = Column(Integer, ForeignKey("enseignes.id"), nullable=True) # type: ignore # NULL = all stores + statut: str = Column(String, nullable=False) # type: ignore # success, error, partial + catalogues_trouves: int = Column(Integer, default=0) # type: ignore + catalogues_nouveaux: int = Column(Integer, default=0) # type: ignore + catalogues_mis_a_jour: int = Column(Integer, default=0) # type: ignore + duree_secondes: float | None = Column(Float, nullable=True) # type: ignore + message_erreur: str | None = Column(Text, nullable=True) # type: ignore + details: str | None = Column(Text, nullable=True) # type: ignore # JSON + + # Relationship + enseigne = relationship("Enseigne", back_populates="scraping_logs") + diff --git a/app/routers/catalogues.py b/app/routers/catalogues.py new file mode 100644 index 0000000..4a03182 --- /dev/null +++ b/app/routers/catalogues.py @@ -0,0 +1,317 @@ +""" +API Router for catalog module +Endpoints for enseignes, catalogues, and admin operations +""" + +import logging +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.orm import Session +from sqlalchemy import func, and_ + +from app.database import get_db +from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog +from app.schemas_catalogues import ( + EnseigneResponse, + CatalogueListResponse, + CatalogueDetailResponse, + CataloguePageResponse, + CataloguesPaginatedResponse, + PaginationMeta, + ScrapingLogResponse, + ScrapingStatsResponse, +) +from app.services.bonial_scraper import scrape_all_enseignes, scrape_enseigne + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/catalogues", tags=["catalogues"]) + + +# === Enseignes Endpoints === + +@router.get("/enseignes", response_model=list[EnseigneResponse]) +async def get_enseignes( + db: Session = Depends(get_db), +): + """Get all active enseignes with catalog counts""" + enseignes = ( + db.query(Enseigne) + .filter(Enseigne.is_active == True) + .order_by(Enseigne.ordre_affichage) + .all() + ) + + # Add catalog counts + result = [] + for enseigne in enseignes: + count = ( + db.query(func.count(Catalogue.id)) + .filter( + and_( + Catalogue.enseigne_id == enseigne.id, + Catalogue.statut == "actif" + ) + ) + .scalar() + ) + + enseigne_data = EnseigneResponse.model_validate(enseigne) + enseigne_data.catalogues_actifs_count = count + result.append(enseigne_data) + + return result + + +# === Catalogues Endpoints === + +@router.get("", response_model=CataloguesPaginatedResponse) +async def get_catalogues( + enseigne_ids: str | None = Query(None, description="Comma-separated enseigne IDs"), + statut: str | None = Query(None, description="Filter by status: actif, termine, erreur, tous"), + date_debut_min: datetime | None = Query(None, description="Start date minimum"), + date_fin_max: datetime | None = Query(None, description="End date maximum"), + recherche: str | None = Query(None, description="Search in titles"), + page: int = Query(1, ge=1, description="Page number"), + limit: int = Query(20, ge=1, le=100, description="Items per page"), + sort: str = Query("date_debut", description="Sort by: date_debut, date_fin, enseigne"), + order: str = Query("desc", description="Order: asc, desc"), + db: Session = Depends(get_db), +): + """Get catalogues with filters and pagination""" + + # Build query + query = db.query(Catalogue) + + # Filter by enseignes + if enseigne_ids: + try: + ids = [int(id_str.strip()) for id_str in enseigne_ids.split(",")] + query = query.filter(Catalogue.enseigne_id.in_(ids)) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid enseigne_ids format") + + # Filter by status + if statut and statut != "tous": + query = query.filter(Catalogue.statut == statut) + + # Filter by dates + if date_debut_min: + query = query.filter(Catalogue.date_debut >= date_debut_min) + if date_fin_max: + query = query.filter(Catalogue.date_fin <= date_fin_max) + + # Search in titles + if recherche: + query = query.filter(Catalogue.titre.ilike(f"%{recherche}%")) + + # Sort + sort_column = { + "date_debut": Catalogue.date_debut, + "date_fin": Catalogue.date_fin, + "enseigne": Catalogue.enseigne_id, + }.get(sort, Catalogue.date_debut) + + if order == "desc": + query = query.order_by(sort_column.desc()) + else: + query = query.order_by(sort_column.asc()) + + # Count total + total = query.count() + + # Paginate + offset = (page - 1) * limit + catalogues = query.offset(offset).limit(limit).all() + + # Build response + data = [] + for catalogue in catalogues: + cat_response = CatalogueListResponse.model_validate(catalogue) + data.append(cat_response) + + pagination = PaginationMeta( + total=total, + page=page, + limit=limit, + pages_total=(total + limit - 1) // limit, + ) + + # Get last scraping time + last_scraping = ( + db.query(ScrapingLog) + .order_by(ScrapingLog.date_execution.desc()) + .first() + ) + + metadata = { + "derniere_mise_a_jour": last_scraping.date_execution if last_scraping else None + } + + return CataloguesPaginatedResponse( + data=data, + pagination=pagination, + metadata=metadata, + ) + + +@router.get("/{catalogue_id}", response_model=CatalogueDetailResponse) +async def get_catalogue_detail( + catalogue_id: int, + db: Session = Depends(get_db), +): + """Get detailed information for a specific catalogue""" + catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first() + + if not catalogue: + raise HTTPException(status_code=404, detail="Catalogue not found") + + return CatalogueDetailResponse.model_validate(catalogue) + + +@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse]) +async def get_catalogue_pages( + catalogue_id: int, + db: Session = Depends(get_db), +): + """Get all pages of a catalogue""" + # Verify catalogue exists + catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first() + + if not catalogue: + raise HTTPException(status_code=404, detail="Catalogue not found") + + # Get pages ordered by page number + pages = ( + db.query(CataloguePage) + .filter(CataloguePage.catalogue_id == catalogue_id) + .order_by(CataloguePage.numero_page) + .all() + ) + + return [CataloguePageResponse.model_validate(page) for page in pages] + + +# === Admin Endpoints === + +@router.post("/admin/scraping/trigger", response_model=dict) +async def trigger_scraping( + enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"), + db: Session = Depends(get_db), +): + """Manually trigger catalog scraping""" + + if enseigne_id: + # Scrape single enseigne + enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first() + + if not enseigne: + raise HTTPException(status_code=404, detail="Enseigne not found") + + # Run scraping asynchronously (in background) + import asyncio + from app.services.bonial_scraper import scrape_enseigne as scrape_ens + + log = await scrape_ens(enseigne, db) + + return { + "success": True, + "message": f"Scraping complete for {enseigne.nom}", + "catalogues_trouves": log.catalogues_trouves, + "catalogues_nouveaux": log.catalogues_nouveaux, + } + else: + # Scrape all enseignes + logs = await scrape_all_enseignes(db) + + total_new = sum(log.catalogues_nouveaux for log in logs) + total_found = sum(log.catalogues_trouves for log in logs) + + return { + "success": True, + "message": f"Scraping complete for all enseignes", + "enseignes_processed": len(logs), + "catalogues_trouves": total_found, + "catalogues_nouveaux": total_new, + } + + +@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse]) +async def get_scraping_logs( + enseigne_id: int | None = Query(None, description="Filter by enseigne"), + statut: str | None = Query(None, description="Filter by status"), + limit: int = Query(50, ge=1, le=200, description="Number of logs to return"), + db: Session = Depends(get_db), +): + """Get scraping execution logs""" + query = db.query(ScrapingLog) + + if enseigne_id: + query = query.filter(ScrapingLog.enseigne_id == enseigne_id) + + if statut: + query = query.filter(ScrapingLog.statut == statut) + + logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all() + + # Add enseigne name + result = [] + for log in logs: + log_dict = ScrapingLogResponse.model_validate(log).model_dump() + if log.enseigne_id: + enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first() + log_dict["enseigne_nom"] = enseigne.nom if enseigne else None + else: + log_dict["enseigne_nom"] = "Toutes" + + result.append(ScrapingLogResponse(**log_dict)) + + return result + + +@router.get("/admin/stats", response_model=ScrapingStatsResponse) +async def get_scraping_stats( + db: Session = Depends(get_db), +): + """Get scraping statistics""" + + # Total catalogues + total_catalogues = db.query(func.count(Catalogue.id)).scalar() + + # Catalogues par enseigne + catalogues_par_enseigne = {} + enseignes = db.query(Enseigne).all() + + for enseigne in enseignes: + count = ( + db.query(func.count(Catalogue.id)) + .filter(Catalogue.enseigne_id == enseigne.id) + .scalar() + ) + catalogues_par_enseigne[enseigne.nom] = count + + # Last scraping time + last_log = ( + db.query(ScrapingLog) + .order_by(ScrapingLog.date_execution.desc()) + .first() + ) + + derniere_mise_a_jour = last_log.date_execution if last_log else None + + # Next execution (6:00 or 18:00) + now = datetime.now() + if now.hour < 6: + prochaine = f"Aujourd'hui à 6:00" + elif now.hour < 18: + prochaine = f"Aujourd'hui à 18:00" + else: + prochaine = f"Demain à 6:00" + + return ScrapingStatsResponse( + total_catalogues=total_catalogues, + catalogues_par_enseigne=catalogues_par_enseigne, + derniere_mise_a_jour=derniere_mise_a_jour, + prochaine_execution=prochaine, + ) diff --git a/app/schemas_catalogues.py b/app/schemas_catalogues.py new file mode 100644 index 0000000..7ee31c0 --- /dev/null +++ b/app/schemas_catalogues.py @@ -0,0 +1,109 @@ +""" +Pydantic schemas for catalog module API +""" + +from datetime import datetime +from pydantic import BaseModel + + +# === Enseigne Schemas === + +class EnseigneResponse(BaseModel): + id: int + nom: str + slug_bonial: str + logo_url: str | None + couleur: str + site_url: str | None + description: str | None + is_active: bool + ordre_affichage: int + + # Statistics + catalogues_actifs_count: int | None = None + + model_config = {"from_attributes": True} + + +# === Catalogue Schemas === + +class CatalogueListResponse(BaseModel): + id: int + enseigne: EnseigneResponse + titre: str + date_debut: datetime + date_fin: datetime + image_couverture_url: str + statut: str + nombre_pages: int + created_at: datetime + + model_config = {"from_attributes": True} + + +class CatalogueDetailResponse(BaseModel): + id: int + enseigne: EnseigneResponse + titre: str + description: str | None + date_debut: datetime + date_fin: datetime + image_couverture_url: str + catalogue_url: str + statut: str + nombre_pages: int + metadonnees: str | None + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} + + +class CataloguePageResponse(BaseModel): + id: int + numero_page: int + image_url: str + image_thumbnail_url: str | None + largeur: int | None + hauteur: int | None + + model_config = {"from_attributes": True} + + +# === Pagination === + +class PaginationMeta(BaseModel): + total: int + page: int + limit: int + pages_total: int + + +class CataloguesPaginatedResponse(BaseModel): + data: list[CatalogueListResponse] + pagination: PaginationMeta + metadata: dict + + +# === Admin Schemas === + +class ScrapingLogResponse(BaseModel): + id: int + date_execution: datetime + enseigne_id: int | None + enseigne_nom: str | None + statut: str + catalogues_trouves: int + catalogues_nouveaux: int + catalogues_mis_a_jour: int + duree_secondes: float | None + message_erreur: str | None + + model_config = {"from_attributes": True} + + +class ScrapingStatsResponse(BaseModel): + total_catalogues: int + catalogues_par_enseigne: dict[str, int] + derniere_mise_a_jour: datetime | None + prochaine_execution: str | None diff --git a/app/services/bonial_scraper.py b/app/services/bonial_scraper.py new file mode 100644 index 0000000..a07026d --- /dev/null +++ b/app/services/bonial_scraper.py @@ -0,0 +1,400 @@ +""" +Bonial.fr Scraper Service + +Scrapes promotional catalogs from Bonial.fr for configured enseignes. +Uses Playwright for browser automation. +""" + +import asyncio +import hashlib +import logging +import re +from datetime import datetime +from typing import Any + +from playwright.async_api import async_playwright, Page, Browser +from sqlalchemy.orm import Session + +from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog + +logger = logging.getLogger(__name__) + +# Bonial selectors based on manual analysis +BONIAL_SELECTORS = { + "catalog_card": "section:has(img[src*='content-media.bonial.biz'])", + "catalog_title": "div.text-dark.text-lg.font-bold, h3, h2", + "catalog_image": "img[src*='content-media.bonial.biz']", + "catalog_link": "a:has-text('Ouvrir le catalogue')", + "cookie_accept": "button:has-text('OK'), button:has-text('Accepter')", +} + +# Date pattern: "mar. 25/11 - lun. 08/12/2025" +DATE_PATTERN = r"(\d{2}/\d{2})\s*-\s*(\d{2}/\d{2}/\d{4})" + + +def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]: + """ + Parse Bonial date format to datetime objects. + + Args: + date_str: Date string like "mar. 25/11 - lun. 08/12/2025" + + Returns: + Tuple of (date_debut, date_fin) + + Raises: + ValueError: If date format is invalid + """ + match = re.search(DATE_PATTERN, date_str) + + if not match: + raise ValueError(f"Invalid date format: {date_str}") + + debut_str, fin_str = match.groups() + + # Extract year from end date + year = fin_str.split("/")[2] + + # Add year to start date + debut_full = f"{debut_str}/{year}" + + # Parse dates + debut = datetime.strptime(debut_full, "%d/%m/%Y") + fin = datetime.strptime(fin_str, "%d/%m/%Y") + + return debut, fin + + +def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str: + """ + Compute SHA256 hash for duplicate detection. + + Args: + enseigne_id: ID of the enseigne + titre: Catalog title + date_debut: Start date + + Returns: + SHA256 hash string + """ + content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}" + return hashlib.sha256(content.encode()).hexdigest() + + +async def accept_cookies(page: Page) -> None: + """Accept cookie consent banner if present.""" + try: + await page.click(BONIAL_SELECTORS["cookie_accept"], timeout=3000) + logger.debug("Accepted cookies") + await asyncio.sleep(1) + except Exception: + logger.debug("No cookie banner found or already accepted") + + +async def scrape_catalog_list( + page: Page, + enseigne: Enseigne +) -> list[dict[str, Any]]: + """ + Scrape the list of catalogs for an ens enseigne from Bonial. + + Args: + page: Playwright page object + enseigne: Enseigne object + + Returns: + List of catalog data dictionaries + """ + url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}" + logger.info(f"Scraping catalog list for {enseigne.nom} from {url}") + + await page.goto(url, wait_until="networkidle") + await accept_cookies(page) + + # Wait for catalog cards to load + try: + await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000) + except Exception as e: + logger.error(f"No catalog cards found for {enseigne.nom}: {e}") + return [] + + # Extract all catalog cards + cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"]) + logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}") + + catalogues = [] + + for idx, card in enumerate(cards): + try: + # Extract title + title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"]) + titre = await title_el.inner_text() if title_el else None + + # Extract dates - look for any element containing date pattern + dates_text = await card.inner_text() + date_match = re.search(DATE_PATTERN, dates_text) + + # Extract image + image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"]) + image_url = await image_el.get_attribute("src") if image_el else None + + # Extract link - try multiple strategies + link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"]) + if not link_el: + # Fallback: click on image + link_el = image_el + + catalogue_url = await link_el.get_attribute("href") if link_el else None + + # Make URL absolute if relative + if catalogue_url and not catalogue_url.startswith("http"): + catalogue_url = f"https://www.bonial.fr{catalogue_url}" + + if all([titre, date_match, image_url, catalogue_url]): + # Parse dates + try: + date_debut, date_fin = parse_bonial_dates(dates_text) + except ValueError as e: + logger.warning(f"Failed to parse dates for catalog {idx}: {e}") + continue + + catalogues.append({ + "titre": titre.strip(), + "date_debut": date_debut, + "date_fin": date_fin, + "image_couverture_url": image_url, + "catalogue_url": catalogue_url, + }) + + logger.debug(f"Extracted catalog: {titre}") + else: + logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={bool(date_match)}, image={bool(image_url)}, url={bool(catalogue_url)}") + + except Exception as e: + logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}") + continue + + logger.info(f"Successfully extracted {len(catalogues)} catalogs for {enseigne.nom}") + return catalogues + + +async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]: + """ + Scrape all pages of a catalog from the Bonial viewer. + + Args: + page: Playwright page object + catalogue_url: URL of the catalog viewer + + Returns: + List of page data dictionaries + """ + logger.info(f"Scraping catalog pages from {catalogue_url}") + + await page.goto(catalogue_url, wait_until="networkidle") + await asyncio.sleep(2) # Wait for viewer to initialize + + # Try to close any popup/modal + try: + await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000) + await asyncio.sleep(1) + except Exception: + pass + + # Find all images from content-media.bonial.biz + images = await page.query_selector_all("img[src*='content-media.bonial.biz']") + + if not images: + logger.warning(f"No images found in catalog viewer: {catalogue_url}") + return [] + + # Extract unique image URLs (filter duplicates and thumbnails) + seen_urls = set() + pages = [] + + for idx, img in enumerate(images): + try: + src = await img.get_attribute("src") + + if src and src not in seen_urls: + # Filter out blurred/low-res images if possible + # Full res images are typically larger + width = await img.evaluate("el => el.naturalWidth") + height = await img.evaluate("el => el.naturalHeight") + + # Skip very small images (likely thumbnails) + if width < 200 or height < 200: + continue + + seen_urls.add(src) + pages.append({ + "numero_page": len(pages) + 1, + "image_url": src, + "largeur": width, + "hauteur": height, + }) + + except Exception as e: + logger.debug(f"Error processing image {idx}: {e}") + continue + + logger.info(f"Found {len(pages)} pages in catalog") + return pages + + +async def scrape_enseigne( + enseigne: Enseigne, + db: Session, + browser: Browser | None = None +) -> ScrapingLog: + """ + Scrape all catalogs for a specific enseigne. + + Args: + enseigne: Enseigne to scrape + db: Database session + browser: Optional existing browser instance + + Returns: + ScrapingLog object with execution details + """ + start_time = datetime.now() + log = ScrapingLog( + enseigne_id=enseigne.id, + statut="error", # Will be updated + catalogues_trouves=0, + catalogues_nouveaux=0, + catalogues_mis_a_jour=0, + ) + + own_browser = browser is None + + try: + # Create browser if not provided + if own_browser: + p = await async_playwright().start() + browser = await p.chromium.launch(headless=True) + + page = await browser.new_page() + + # Set realistic user agent + await page.set_extra_http_headers({ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + }) + + # Scrape catalog list + catalogues_data = await scrape_catalog_list(page, enseigne) + log.catalogues_trouves = len(catalogues_data) + + if not catalogues_data: + log.statut = "success" + log.message_erreur = "No catalogs found (may be normal)" + await page.close() + return log + + # Process each catalog + for cat_data in catalogues_data: + try: + # Compute hash for duplicate detection + content_hash = compute_catalog_hash( + enseigne.id, + cat_data["titre"], + cat_data["date_debut"] + ) + + # Check if catalog already exists + existing = db.query(Catalogue).filter_by(content_hash=content_hash).first() + + if existing: + logger.debug(f"Catalog already exists: {cat_data['titre']}") + continue + + # Scrape catalog pages + await asyncio.sleep(3) # Respectful delay + pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"]) + + # Create new catalog + catalogue = Catalogue( + enseigne_id=enseigne.id, + titre=cat_data["titre"], + date_debut=cat_data["date_debut"], + date_fin=cat_data["date_fin"], + image_couverture_url=cat_data["image_couverture_url"], + catalogue_url=cat_data["catalogue_url"], + nombre_pages=len(pages_data), + content_hash=content_hash, + statut="actif" if cat_data["date_fin"] >= datetime.now() else "termine", + ) + + db.add(catalogue) + db.flush() # Get catalogue.id + + # Add pages + for page_data in pages_data: + cat_page = CataloguePage( + catalogue_id=catalogue.id, + **page_data + ) + db.add(cat_page) + + log.catalogues_nouveaux += 1 + logger.info(f"Created catalog: {cat_data['titre']} ({len(pages_data)} pages)") + + except Exception as e: + logger.error(f"Error processing catalog {cat_data.get('titre')}: {e}") + continue + + await page.close() + + # Update status + log.statut = "success" if log.catalogues_nouveaux > 0 else "partial" + db.commit() + + except Exception as e: + logger.error(f"Error scraping enseigne {enseigne.nom}: {e}") + log.statut = "error" + log.message_erreur = str(e) + db.rollback() + + finally: + if own_browser and browser: + await browser.close() + + # Record duration + log.duree_secondes = (datetime.now() - start_time).total_seconds() + db.add(log) + db.commit() + + return log + + +async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]: + """ + Scrape all active enseignes. + + Args: + db: Database session + + Returns: + List of ScrapingLog objects + """ + enseignes = db.query(Enseigne).filter_by(is_active=True).all() + logger.info(f"Starting scraping for {len(enseignes)} active enseignes") + + logs = [] + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + + for enseigne in enseignes: + try: + await asyncio.sleep(2) # Delay between enseignes + log = await scrape_enseigne(enseigne, db, browser) + logs.append(log) + except Exception as e: + logger.error(f"Failed to scrape {enseigne.nom}: {e}") + + await browser.close() + + logger.info(f"Scraping complete: {len(logs)} enseignes processed") + return logs diff --git a/app/services/scheduler.py b/app/services/scheduler.py new file mode 100644 index 0000000..4ef299a --- /dev/null +++ b/app/services/scheduler.py @@ -0,0 +1,62 @@ +""" +Scheduler for automated catalog scraping + +Runs scraping jobs twice daily (6h and 18h) for all active enseignes. +""" + +import logging +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.cron import CronTrigger + +from app.database import SessionLocal +from app.services.bonial_scraper import scrape_all_enseignes + +logger = logging.getLogger(__name__) + +# Global scheduler instance +scheduler = AsyncIOScheduler() + + +async def scheduled_scrape_job(): + """Scheduled job to scrape all active enseignes""" + logger.info("Starting scheduled catalog scraping job") + + db = SessionLocal() + try: + logs = await scrape_all_ense ignes(db) + + total_new = sum(log.catalogues_nouveaux for log in logs) + total_found = sum(log.catalogues_trouves for log in logs) + + logger.info( + f"Scheduled scraping complete: " + f"{len(logs)} enseignes processed, " + f"{total_found} catalogues found, " + f"{total_new} new catalogues created" + ) + except Exception as e: + logger.error(f"Error in scheduled scraping job: {e}") + finally: + db.close() + + +def start_scheduler(): + """Start the scheduler with configured jobs""" + # Schedule scraping twice daily at 6:00 and 18:00 + scheduler.add_job( + scheduled_scrape_job, + CronTrigger(hour="6,18", minute=0), + id="catalog_scraping", + name="Scrape Bonial catalogs", + replace_existing=True, + ) + + scheduler.start() + logger.info("Catalog scraping scheduler started (runs at 6:00 and 18:00)") + + +def stop_scheduler(): + """Stop the scheduler""" + if scheduler.running: + scheduler.shutdown() + logger.info("Catalog scraping scheduler stopped") diff --git a/app/services/seed_enseignes.py b/app/services/seed_enseignes.py new file mode 100644 index 0000000..ab78cb7 --- /dev/null +++ b/app/services/seed_enseignes.py @@ -0,0 +1,128 @@ +""" +Seed default enseignes (stores) for the catalog module +""" + +import logging +from sqlalchemy.orm import Session + +from app.models import Enseigne + +logger = logging.getLogger(__name__) + +# Data for the 9 enseignes +ENSEIGNES_DATA = [ + { + "nom": "Gifi", + "slug_bonial": "Gifi", + "couleur": "#E30613", + "site_url": "https://www.gifi.fr", + "description": "Décoration, maison, bazar", + "ordre_affichage": 1, + }, + { + "nom": "Action", + "slug_bonial": "Action", + "couleur": "#0066B3", + "site_url": "https://www.action.com/fr-fr/", + "description": "Discount non-alimentaire", + "ordre_affichage": 2, + }, + { + "nom": "Centrakor", + "slug_bonial": "Centrakor", + "couleur": "#E94E1B", + "site_url": "https://www.centrakor.com", + "description": "Décoration, maison", + "ordre_affichage": 3, + }, + { + "nom": "La Foir'Fouille", + "slug_bonial": "La-Foir-Fouille", + "couleur": "#009639", + "site_url": "https://www.lafoirfouille.fr", + "description": "Bazar, décoration", + "ordre_affichage": 4, + }, + { + "nom": "Stokomani", + "slug_bonial": "Stokomani", + "couleur": "#FF6600", + "site_url": "https://www.stokomani.fr", + "description": "Déstockage textile et maison", + "ordre_affichage": 5, + }, + { + "nom": "B&M", + "slug_bonial": "BM", + "couleur": "#D4145A", + "site_url": "https://bmstores.fr", + "description": "Discount britannique", + "ordre_affichage": 6, + }, + { + "nom": "L'Incroyable", + "slug_bonial": "L-incroyable", + "couleur": "#8B0000", + "site_url": "https://www.lincroyable.fr", + "description": "Décoration et mobilier discount (Groupe Althys, siège à Denain)", + "ordre_affichage": 7, + }, + { + "nom": "Bazarland", + "slug_bonial": "Bazarland", + "couleur": "#FFCC00", + "site_url": None, + "description": "Bazar discount", + "ordre_affichage": 8, + }, + { + "nom": "Noz", + "slug_bonial": "Noz", + "couleur": "#003366", + "site_url": "https://www.noz.fr", + "description": "Déstockage généraliste", + "ordre_affichage": 9, + }, +] + + +def seed_enseignes(db: Session) -> int: + """ + Seed the default enseignes if they don't exist. + + Returns: + Number of enseignes created + """ + created = 0 + + for data in ENSEIGNES_DATA: + # Check if enseigne already exists + existing = db.query(Enseigne).filter_by(slug_bonial=data["slug_bonial"]).first() + + if not existing: + enseigne = Enseigne(**data) + db.add(enseigne) + created += 1 + logger.info(f"Created enseigne: {data['nom']}") + else: + logger.debug(f"Enseigne already exists: {data['nom']}") + + db.commit() + logger.info(f"Seeding complete: {created} enseigne(s) created") + + return created + + +def get_enseigne_by_slug(db: Session, slug: str) -> Enseigne | None: + """Get an enseigne by its Bonial slug""" + return db.query(Enseigne).filter_by(slug_bonial=slug).first() + + +def get_all_active_enseignes(db: Session) -> list[Enseigne]: + """Get all active enseignes ordered by display order""" + return ( + db.query(Enseigne) + .filter_by(is_active=True) + .order_by(Enseigne.ordre_affichage) + .all() + ) diff --git a/docs/API_CATALOGUES.md b/docs/API_CATALOGUES.md new file mode 100644 index 0000000..f0f4876 --- /dev/null +++ b/docs/API_CATALOGUES.md @@ -0,0 +1,329 @@ +# API Documentation - Module Catalogues Bonial + +## Vue d'ensemble + +Le module Catalogues permet de consulter les catalogues promotionnels des enseignes discount scrapers depuis Bonial.fr. + +**Base URL**: `/api/catalogues` + +## Endpoints Publics + +### 1. Liste des Enseignes + +`GET /api/catalogues/enseignes` + +Retourne la liste de toutes les enseignes actives avec le nombre de catalogues en cours. + +**Réponse** : `200 OK` +```json +[ + { + "id": 1, + "nom": "Gifi", + "slug_bonial": "Gifi", + "logo_url": null, + "couleur": "#E30613", + "site_url": "https://www.gifi.fr", + "description": "Décoration, maison, bazar", + "is_active": true, + "ordre_affichage": 1, + "catalogues_actifs_count": 3 + } +] +``` + +--- + +### 2. Liste des Catalogues (Paginée) + +`GET /api/catalogues` + +Retourne la liste paginée des catalogues avec filtres. + +**Paramètres Query** : +- `page` (int, default=1): Numéro de page +- `limit` (int, default=20, max=100): Nombre d'éléments par page +- `enseigne_ids` (string, optional): IDs d'enseignes séparés par virgule (ex: `1,2,3`) +- `statut` (string, optional): Filtre par statut (`actif`, `termine`, `tous`) +- `date_debut_min` (datetime, optional): Date de début minimum +- `date_fin_max` (datetime, optional): Date de fin maximum +- `recherche` (string, optional): Recherche dans les titres +- `sort` (string, default=`date_debut`): Tri (`date_debut`, `date_fin`, `enseigne`) +- `order` (string, default=`desc`): Ordre (`asc`, `desc`) + +**Réponse** : `200 OK` +```json +{ + "data": [ + { + "id": 42, + "enseigne": { + "id": 1, + "nom": "Gifi", + "couleur": "#E30613", + ... + }, + "titre": "Catalogue Gifi Noël 2024", + "date_debut": "2024-11-25T00:00:00", + "date_fin": "2024-12-24T23:59:59", + "image_couverture_url": "https://content-media.bonial.biz/...", + "statut": "actif", + "nombre_pages": 24, + "created_at": "2024-11-29T06:00:00" + } + ], + "pagination": { + "total": 156, + "page": 1, + "limit": 20, + "pages_total": 8 + }, + "metadata": { + "derniere_mise_a_jour": "2024-11-29T06:15:32" + } +} +``` + +--- + +### 3. Détail d'un Catalogue + +`GET /api/catalogues/{catalogue_id}` + +Retourne les informations détaillées d'un catalogue. + +**Paramètres Path** : +- `catalogue_id` (int): ID du catalogue + +**Réponse** : `200 OK` +```json +{ + "id": 42, + "enseigne": { + "id": 1, + "nom": "Gifi", + ... + }, + "titre": "Catalogue Gifi Noël 2024", + "description": null, + "date_debut": "2024-11-25T00:00:00", + "date_fin": "2024-12-24T23:59:59", + "image_couverture_url": "https://content-media.bonial.biz/...", + "catalogue_url": "https://www.bonial.fr/...", + "statut": "actif", + "nombre_pages": 24, + "metadonnees": null, + "created_at": "2024-11-29T06:00:00", + "updated_at": "2024-11-29T06:00:00" +} +``` + +**Erreurs** : +- `404 Not Found`: Catalogue introuvable + +--- + +### 4. Pages d'un Catalogue + +`GET /api/catalogues/{catalogue_id}/pages` + +Retourne toutes les pages d'un catalogue, ordonnées par numéro. + +**Paramètres Path** : +- `catalogue_id` (int): ID du catalogue + +**Réponse** : `200 OK` +```json +[ + { + "id": 128, + "numero_page": 1, + "image_url": "https://content-media.bonial.biz/.../page_1.jpg", + "image_thumbnail_url": null, + "largeur": 1200, + "hauteur": 1600 + }, + { + "id": 129, + "numero_page": 2, + "image_url": "https://content-media.bonial.biz/.../page_2.jpg", + "image_thumbnail_url": null, + "largeur": 1200, + "hauteur": 1600 + } +] +``` + +**Erreurs** : +- `404 Not Found`: Catalogue introuvable + +--- + +## Endpoints Admin (Authentification requise) + +### 5. Déclencher Scraping Manuel + +`POST /api/catalogues/admin/scraping/trigger` + +Déclenche un scraping manuel des catalogues Bonial. + +**Authentification** : Requise (JWT token + rôle `admin`) + +**Paramètres Query** : +- `enseigne_id` (int, optional): ID de l'enseigne à scraper. Si omis, scrape toutes les enseignes actives. + +**Réponse** : `200 OK` +```json +{ + "success": true, + "message": "Scraping complete for Gifi", + "catalogues_trouves": 4, + "catalogues_nouveaux": 2 +} +``` + +Ou pour toutes les enseignes : +```json +{ + "success": true, + "message":" Scraping complete for all enseignes", + "enseignes_processed": 9, + "catalogues_trouves": 38, + "catalogues_nouveaux": 12 +} +``` + +**Erreurs** : +- `403 Forbidden`: Accès admin requis +- `404 Not Found`: Enseigne introuvable + +--- + +### 6. Historique des Scraping + +`GET /api/catalogues/admin/scraping/logs` + +Retourne l'historique des exécutions de scraping. + +**Authentification** : Requise (JWT token + rôle `admin`) + +**Paramètres Query** : +- `limit` (int, default=50, max=200): Nombre de logs à retourner +- `enseigne_id` (int, optional): Filtre par enseigne +- `statut` (string, optional): Filtre par statut (`success`, `error`, `partial`) + +**Réponse** : `200 OK` +```json +[ + { + "id": 15, + "date_execution": "2024-11-29T06:00:15", + "enseigne_id": 1, + "enseigne_nom": "Gifi", + "statut": "success", + "catalogues_trouves": 4, + "catalogues_nouveaux": 2, + "catalogues_mis_a_jour": 0, + "duree_secondes": 45.2, + "message_erreur": null + } +] +``` + +--- + +### 7. Statistiques de Scraping + +`GET /api/catalogues/admin/stats` + +Retourne les statistiques globales du module catalogues. + +**Authentification** : Requise (JWT token + rôle `admin`) + +**Réponse** : `200 OK` +```json +{ + "total_catalogues": 156, + "catalogues_par_enseigne": { + "Gifi": 18, + "Action": 22, + "Centrakor": 15, + "La Foir'Fouille": 12, + "Stokomani": 19, + "B&M": 14, + "L'Incroyable": 11, + "Bazarland": 8, + "Noz": 17 + }, + "derniere_mise_a_jour": "2024-11-29T06:15:32", + "prochaine_execution": "Aujourd'hui à 18:00" +} +``` + +--- + +## Erreurs Communes + +### 400 Bad Request +```json +{ + "detail": "Invalid enseigne_ids format" +} +``` + +### 403 Forbidden +```json +{ + "detail": "Admin access required" +} +``` + +### 404 Not Found +```json +{ + "detail": "Catalogue not found" +} +``` + +### 500 Internal Server Error +```json +{ + "detail": "Internal server error" +} +``` + +--- + +## Scheduler Automatique + +Le scraping est exécuté automatiquement **2 fois par jour** : +- **6h00** : Scraping matinal +- **18h00** : Scraping du soir + +Les catalogues sont automatiquement marqués comme `termine` lorsque leur `date_fin` est dépassée. + +--- + +## Notes d'Implémentation + +### Détection de Doublons + +Le système utilise un hash SHA-256 basé sur : +- `enseigne_id` +- `titre` +- `date_debut` + +Cela évite de créer des doublons même si le catalogue est re-scrapé. + +### Performance + +- Pagination obligatoire (max 100 items/page) +- Index DB sur `enseigne_id`, `date_debut`, `date_fin`, `statut` +- Scraping asynchrone (n'impacte pas les performances API) + +### Formats de Date + +Toutes les dates sont en **ISO 8601** avec timezone UTC : +``` +2024-11-29T06:15:32+00:00 +``` diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 2d11e0e..eef1c9f 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -5,6 +5,7 @@ import Layout from '@/components/layout/Layout' import Dashboard from '@/pages/Dashboard' import Search from '@/pages/Search' import MultiSearch from '@/pages/MultiSearch' +import Catalogues from '@/pages/Catalogues' import Login from '@/pages/Login' import Admin from '@/pages/Admin' import { useTheme } from '@/hooks/use-theme' @@ -77,6 +78,7 @@ function AppRoutes() { } /> } /> } /> + } /> { + loadEnseignes(); + loadStats(); + loadLogs(); + }, []); + + const loadEnseignes = async () => { + try { + const response = await axios.get('/api/catalogues/enseignes'); + setEnseignes(response.data); + } catch (error) { + console.error('Error loading enseignes:', error); + } + }; + + const loadStats = async () => { + try { + const response = await axios.get(`${API_BASE}/stats`); + setStats(response.data); + } catch (error) { + console.error('Error loading stats:', error); + } + }; + + const loadLogs = async () => { + setLoading(true); + try { + const response = await axios.get(`${API_BASE}/scraping/logs?limit=20`); + setLogs(response.data); + } catch (error) { + console.error('Error loading logs:', error); + toast.error('Erreur lors du chargement des logs'); + } finally { + setLoading(false); + } + }; + + const triggerScraping = async (enseigneId = null) => { + setScraping(true); + try { + const params = enseigneId ? { enseigne_id: enseigneId } : {}; + const response = await axios.post(`${API_BASE}/scraping/trigger`, null, { params }); + + toast.success(response.data.message); + toast.info(`${response.data.catalogues_nouveaux} nouveaux catalogues trouvés`); + + // Reload data + await Promise.all([loadStats(), loadLogs()]); + } catch (error) { + console.error('Error triggering scraping:', error); + toast.error('Erreur lors du déclenchement du scraping'); + } finally { + setScraping(false); + } + }; + + const getStatusIcon = (status) => { + switch (status) { + case 'success': + return ; + case 'error': + return ; + case 'partial': + return ; + default: + return ; + } + }; + + const formatDate = (dateString) => { + const date = new Date(dateString); + return date.toLocaleString('fr-FR'); + }; + + return ( +
+
+

Administration Catalogues

+

Gérer le scraping des catalogues Bonial

+
+ + {/* Stats Cards */} + {stats && ( +
+
+

Total Catalogues

+

{stats.total_catalogues}

+
+
+

Dernière Mise à Jour

+

+ {stats.derniere_mise_a_jour + ? formatDate(stats.derniere_mise_a_jour) + : 'Jamais'} +

+
+
+

Prochaine Exécution

+

{stats.prochaine_execution}

+
+
+ )} + + {/* Manual Scraping */} +
+

Scraping Manuel

+
+ + +
+

Ou scraper une enseigne spécifique:

+
+ {enseignes.map(enseigne => ( + + ))} +
+
+
+
+ + {/* Catalogues par Enseigne */} + {stats && ( +
+

Catalogues par Enseigne

+
+ {Object.entries(stats.catalogues_par_enseigne).map(([nom, count]) => ( +
+ {nom} + {count} +
+ ))} +
+
+ )} + + {/* Scraping Logs */} +
+
+

Historique des Scraping

+ +
+ + {loading ? ( +
+ +
+ ) : logs.length === 0 ? ( +

Aucun log disponible

+ ) : ( +
+ {logs.map(log => ( +
+
+ {getStatusIcon(log.statut)} +
+
+
+ {log.enseigne_nom || 'Toutes'} + + {formatDate(log.date_execution)} + +
+
+ {log.catalogues_trouves} trouvé(s) • {log.catalogues_nouveaux} nouveau(x) + {log.duree_secondes && ` • ${log.duree_secondes.toFixed(1)}s`} +
+ {log.message_erreur && ( +
+ {log.message_erreur} +
+ )} +
+
+ ))} +
+ )} +
+
+ ); +} diff --git a/frontend/src/components/layout/Layout.jsx b/frontend/src/components/layout/Layout.jsx index 46c4dbe..04b038e 100644 --- a/frontend/src/components/layout/Layout.jsx +++ b/frontend/src/components/layout/Layout.jsx @@ -1,7 +1,7 @@ import React, { useState } from 'react'; import { Link, useLocation, useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; -import { LayoutDashboard, Search, TrendingUp, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react'; +import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet'; import { cn } from '@/lib/utils'; @@ -23,6 +23,7 @@ const Layout = ({ children, theme, toggleTheme }) => { { icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' }, { icon: Search, label: t('nav.search') || 'Recherche', path: '/search' }, { icon: TrendingUp, label: 'Comparateur', path: '/compare' }, + { icon: BookOpen, label: 'Catalogues', path: '/catalogues' }, ]; // Add admin link for admin users diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx index 1d7af6f..0971f0e 100644 --- a/frontend/src/pages/Admin.jsx +++ b/frontend/src/pages/Admin.jsx @@ -7,7 +7,7 @@ import { Settings as SettingsIcon, Cpu, Clock, Bell, Globe, ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2, Sparkles, Eye, Code, Brain, DollarSign, Filter, - TrendingDown, CheckCircle2, Package, Bug, Download + TrendingDown, CheckCircle2, Package, Bug, Download, BookOpen } from 'lucide-react'; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'; import { Label } from '@/components/ui/label'; @@ -18,6 +18,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@ import { Slider } from '@/components/ui/slider'; import { cn } from '@/lib/utils'; import { useAuth } from '@/hooks/use-auth'; +import CatalogueAdmin from '@/components/dashboard/CatalogueAdmin'; const API_URL = '/api'; @@ -495,6 +496,7 @@ export default function Admin() { { id: 'sites', label: 'Sites', icon: Globe }, { id: 'jobs', label: 'Jobs', icon: Clock }, { id: 'notifications', label: 'Notifications', icon: Bell }, + { id: 'catalogues', label: 'Catalogues', icon: BookOpen }, { id: 'debug', label: 'Debug', icon: Bug }, ]; @@ -1120,6 +1122,9 @@ export default function Admin() { )} - + + {/* Catalogues Tab */} + {activeTab === 'catalogues' && } + ); } diff --git a/frontend/src/pages/Catalogues.jsx b/frontend/src/pages/Catalogues.jsx new file mode 100644 index 0000000..d219621 --- /dev/null +++ b/frontend/src/pages/Catalogues.jsx @@ -0,0 +1,323 @@ +import React, { useEffect, useState } from 'react'; +import { toast } from 'sonner'; +import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye } from 'lucide-react'; +import axios from 'axios'; + +const API_BASE = '/api/catalogues'; + +export default function Catalogues() { + const [loading, setLoading] = useState(true); + const [enseignes, setEnseignes] = useState([]); + const [catalogues, setCatalogues] = useState([]); + const [selectedEnseigne, setSelectedEnseigne] = useState(null); + const [pagination, setPagination] = useState({ page: 1, limit: 12, total: 0 }); + const [selectedCatalogue, setSelectedCatalogue] = useState(null); + const [cataloguePages, setCataloguePages] = useState([]); + const [searchQuery, setSearchQuery] = useState(''); + + useEffect(() => { + loadEnseignes(); + loadCatalogues(); + }, []); + + useEffect(() => { + loadCatalogues(); + }, [pagination.page, selectedEnseigne, searchQuery]); + + const loadEnseignes = async () => { + try { + const response = await axios.get(`${API_BASE}/enseignes`); + setEnseignes(response.data); + } catch (error) { + console.error('Error loading enseignes:', error); + toast.error('Erreur lors du chargement des enseignes'); + } + }; + + const loadCatalogues = async () => { + setLoading(true); + try { + const params = { + page: pagination.page, + limit: pagination.limit, + statut: 'actif', + }; + + if (selectedEnseigne) { + params.enseigne_ids = selectedEnseigne.id.toString(); + } + + if (searchQuery) { + params.recherche = searchQuery; + } + + const response = await axios.get(API_BASE, { params }); + setCatalogues(response.data.data); + setPagination(prev => ({ ...prev, total: response.data.pagination.total })); + } catch (error) { + console.error('Error loading catalogues:', error); + toast.error('Erreur lors du chargement des catalogues'); + } finally { + setLoading(false); + } + }; + + const loadCataloguePages = async (catalogueId) => { + try { + const response = await axios.get(`${API_BASE}/${catalogueId}/pages`); + setCataloguePages(response.data); + } catch (error) { + console.error('Error loading pages:', error); + toast.error('Erreur lors du chargement des pages'); + } + }; + + const openCatalogue = (catalogue) => { + setSelectedCatalogue(catalogue); + loadCataloguePages(catalogue.id); + }; + + const closeCatalogue = () => { + setSelectedCatalogue(null); + setCataloguePages([]); + }; + + const formatDate = (dateString) => { + const date = new Date(dateString); + return date.toLocaleDateString('fr-FR', { day: 'numeric', month: 'short', year: 'numeric' }); + }; + + const isValidNow = (catalogue) => { + const now = new Date(); + const debut = new Date(catalogue.date_debut); + const fin = new Date(catalogue.date_fin); + return now >= debut && now <= fin; + }; + + return ( +
+ {/* Header */} +
+

Catalogues Promotionnels

+

+ Consultez les catalogues et prospectus des enseignes discount +

+
+ + {/* Search Bar */} +
+
+ + { + setSearchQuery(e.target.value); + setPagination(prev => ({ ...prev, page: 1 })); + }} + /> +
+
+ + {/* Enseignes Filter */} +
+
+ + {enseignes.map(enseigne => ( + + ))} +
+
+ + {/* Catalogues Grid */} + {loading ? ( +
+ +
+ ) : catalogues.length === 0 ? ( +
+ +

Aucun catalogue trouvé

+

+ {searchQuery + ? 'Aucun catalogue ne correspond à votre recherche' + : 'Aucun catalogue disponible pour le moment'} +

+
+ ) : ( + <> +
+ {catalogues.map(catalogue => ( +
openCatalogue(catalogue)} + > + {/* Cover Image */} +
+ {catalogue.titre} +
+
+ +

Voir le catalogue

+
+
+ {isValidNow(catalogue) && ( +
+ + En cours + +
+ )} +
+ + {/* Content */} +
+ {/* Enseigne Badge */} +
+ {catalogue.enseigne.nom} +
+ + {/* Title */} +

+ {catalogue.titre} +

+ + {/* Dates */} +
+ + + {formatDate(catalogue.date_debut)} - {formatDate(catalogue.date_fin)} + +
+ + {/* Pages */} +
+ + {catalogue.nombre_pages} pages +
+
+
+ ))} +
+ + {/* Pagination */} + {pagination.total > pagination.limit && ( +
+ + + Page {pagination.page} sur {Math.ceil(pagination.total / pagination.limit)} + + +
+ )} + + )} + + {/* Catalogue Viewer Modal */} + {selectedCatalogue && ( +
+
+ {/* Header */} +
+
+

{selectedCatalogue.titre}

+

+ {selectedCatalogue.enseigne.nom} • {cataloguePages.length} pages +

+
+ +
+ + {/* Pages Viewer */} +
+ {cataloguePages.length === 0 ? ( +
+ +
+ ) : ( +
+ {cataloguePages.map(page => ( +
window.open(page.image_url, '_blank')} + > + {`Page +
+ Page {page.numero_page} +
+
+ ))} +
+ )} +
+
+
+ )} +
+ ); +} diff --git a/setup_bonial.ps1 b/setup_bonial.ps1 new file mode 100644 index 0000000..ddc8e2b --- /dev/null +++ b/setup_bonial.ps1 @@ -0,0 +1,72 @@ +# Setup and Migration Script for Bonial Module +# +# Run this script to set up the Bonial catalog module + +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host " BONIAL CATALOG MODULE - SETUP SCRIPT" -ForegroundColor Cyan +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host "" + +# Check if Docker is running +Write-Host "[1/3] Checking Docker..." -ForegroundColor Yellow +try { + $dockerCheck = docker ps 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Host " ERROR: Docker is not running!" -ForegroundColor Red + Write-Host " Please start Docker Desktop and try again." -ForegroundColor Red + exit 1 + } + Write-Host " OK - Docker is running" -ForegroundColor Green +} catch { + Write-Host " ERROR: Docker not found!" -ForegroundColor Red + Write-Host " Please install Docker Desktop." -ForegroundColor Red + exit 1 +} + +# Run Alembic migration inside container +Write-Host "" +Write-Host "[2/3] Running database migration..." -ForegroundColor Yellow +try { + docker compose exec -T priceflow alembic upgrade head + if ($LASTEXITCODE -eq 0) { + Write-Host " OK - Migration completed successfully" -ForegroundColor Green + } else { + Write-Host " WARNING: Migration may have failed" -ForegroundColor Yellow + Write-Host " Tables may already exist - this is normal" -ForegroundColor Yellow + } +} catch { + Write-Host " ERROR: Could not run migration" -ForegroundColor Red + Write-Host " Error: $_" -ForegroundColor Red +} + +# Restart application to trigger seeding +Write-Host "" +Write-Host "[3/3] Restarting application to seed enseignes..." -ForegroundColor Yellow +try { + docker compose restart priceflow + if ($LASTEXITCODE -eq 0) { + Write-Host " OK - Application restarted" -ForegroundColor Green + Start-Sleep -Seconds 3 + Write-Host " Check logs for: 'X enseigne(s) créée(s)'" -ForegroundColor Cyan + } else { + Write-Host " ERROR: Could not restart application" -ForegroundColor Red + } +} catch { + Write-Host " ERROR: $_" -ForegroundColor Red +} + +Write-Host "" +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host " SETUP COMPLETE!" -ForegroundColor Green +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Next steps:" -ForegroundColor Yellow +Write-Host " 1. Check application logs:" -ForegroundColor White +Write-Host " docker compose logs -f priceflow" -ForegroundColor Gray +Write-Host "" +Write-Host " 2. Run verification script:" -ForegroundColor White +Write-Host " docker compose exec priceflow python verify_bonial.py" -ForegroundColor Gray +Write-Host "" +Write-Host " 3. Test API endpoints:" -ForegroundColor White +Write-Host " Invoke-WebRequest http://localhost:8555/api/catalogues/enseignes" -ForegroundColor Gray +Write-Host "" diff --git a/verify_bonial.py b/verify_bonial.py new file mode 100644 index 0000000..90ba055 --- /dev/null +++ b/verify_bonial.py @@ -0,0 +1,294 @@ +""" +Verification Script for Bonial Catalog Module + +Tests database setup, scraper functionality, and API endpoints. +""" + +import asyncio +import logging +import sys +from datetime import datetime + +import httpx +from sqlalchemy import inspect, text + +from app.database import SessionLocal, engine +from app.models import Catalogue, CataloguePage, Enseigne, ScrapingLog +from app.services.bonial_scraper import scrape_enseigne +from app.services.seed_enseignes import seed_enseignes + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +API_BASE_URL = "http://localhost:8555/api" + + +def check_database_tables(): + """Verify that catalog module tables exist.""" + logger.info("=" * 60) + logger.info("Phase 1: Checking Database Tables") + logger.info("=" * 60) + + inspector = inspect(engine) + required_tables = [ + "enseignes", + "catalogues", + "catalogue_pages", + "scraping_logs", + ] + + existing_tables = inspector.get_table_names() + + all_exist = True + for table in required_tables: + exists = table in existing_tables + status = "✅" if exists else "❌" + logger.info(f"{status} Table '{table}': {'EXISTS' if exists else 'MISSING'}") + if not exists: + all_exist = False + + if all_exist: + logger.info("\n✅ All required tables exist!\n") + return True + else: + logger.error("\n❌ Some tables are missing. Run migration: alembic upgrade head\n") + return False + + +def check_enseignes_seeding(): + """Verify that enseignes are seeded.""" + logger.info("=" * 60) + logger.info("Phase 2: Checking Enseignes Seeding") + logger.info("=" * 60) + + db = SessionLocal() + try: + count = db.query(Enseigne).count() + logger.info(f"Found {count} enseignes in database") + + if count == 0: + logger.info("Seeding enseignes...") + created = seed_enseignes(db) + logger.info(f"✅ Created {created} enseignes") + count = created + + if count >= 9: + logger.info("\n✅ All 9 enseignes are seeded!\n") + + # Display enseignes + enseignes = db.query(Enseigne).order_by(Enseigne.ordre_affichage).all() + for ens in enseignes: + active = "✅" if ens.is_active else "⚠️" + logger.info(f" {active} {ens.ordre_affichage}. {ens.nom} (slug: {ens.slug_bonial})") + + return True + else: + logger.warning(f"\n⚠️ Expected 9 enseignes, found {count}\n") + return False + + finally: + db.close() + + +async def test_scraper(): + """Test the Bonial scraper on one enseigne.""" + logger.info("=" * 60) + logger.info("Phase 3: Testing Bonial Scraper (Gifi)") + logger.info("=" * 60) + + db = SessionLocal() + try: + # Get Gifi enseigne + gifi = db.query(Enseigne).filter_by(slug_bonial="Gifi").first() + + if not gifi: + logger.error("❌ Gifi enseigne not found") + return False + + logger.info(f"Testing scraper for: {gifi.nom}") + logger.info("This may take 30-60 seconds...") + + # Run scraper + log = await scrape_enseigne(gifi, db) + + # Display results + logger.info(f"\nScraping completed:") + logger.info(f" Status: {log.statut}") + logger.info(f" Catalogues found: {log.catalogues_trouves}") + logger.info(f" New catalogues: {log.catalogues_nouveaux}") + logger.info(f" Duration: {log.duree_secondes:.2f}s") + + if log.message_erreur: + logger.warning(f" Error: {log.message_erreur}") + + if log.statut in ["success", "partial"] and log.catalogues_trouves > 0: + logger.info("\n✅ Scraper is working!\n") + + # Display sample catalog + cat = db.query(Catalogue).filter_by(enseigne_id=gifi.id).first() + if cat: + logger.info(f"Sample catalog:") + logger.info(f" Titre: {cat.titre}") + logger.info(f" Dates: {cat.date_debut.date()} → {cat.date_fin.date()}") + logger.info(f" Pages: {cat.nombre_pages}") + + return True + else: + logger.error("\n❌ Scraper failed or found no catalogs\n") + return False + + except Exception as e: + logger.error(f"❌ Error testing scraper: {e}") + return False + finally: + db.close() + + +async def test_api_endpoints(): + """Test API endpoints.""" + logger.info("=" * 60) + logger.info("Phase 4: Testing API Endpoints") + logger.info("=" * 60) + + async with httpx.AsyncClient(timeout=30.0) as client: + # Test 1: Get enseignes + logger.info("\n1. Testing GET /api/catalogues/enseignes") + try: + response = await client.get(f"{API_BASE_URL}/catalogues/enseignes") + if response.status_code == 200: + enseignes = response.json() + logger.info(f" ✅ Status 200 - Found {len(enseignes)} enseignes") + if enseignes: + logger.info(f" Sample: {enseignes[0]['nom']} ({enseignes[0]['catalogues_actifs_count']} catalogues)") + else: + logger.error(f" ❌ Status {response.status_code}") + return False + except Exception as e: + logger.error(f" ❌ Error: {e}") + return False + + # Test 2: Get catalogues + logger.info("\n2. Testing GET /api/catalogues") + try: + response = await client.get(f"{API_BASE_URL}/catalogues?page=1&limit=5") + if response.status_code == 200: + data = response.json() + catalogues = data.get("data", []) + pagination = data.get("pagination", {}) + logger.info(f" ✅ Status 200 - Found {pagination.get('total', 0)} catalogues") + logger.info(f" Page: {pagination.get('page')}/{pagination.get('pages_total')}") + if catalogues: + logger.info(f" Sample: {catalogues[0]['titre']}") + else: + logger.error(f" ❌ Status {response.status_code}") + return False + except Exception as e: + logger.error(f" ❌ Error: {e}") + return False + + # Test 3: Get catalogue detail + logger.info("\n3. Testing GET /api/catalogues/{id}") + try: + # Get first catalog ID + db = SessionLocal() + cat = db.query(Catalogue).first() + db.close() + + if cat: + response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}") + if response.status_code == 200: + detail = response.json() + logger.info(f" ✅ Status 200 - Catalogue: {detail['titre']}") + logger.info(f" Pages: {detail['nombre_pages']}") + else: + logger.error(f" ❌ Status {response.status_code}") + return False + else: + logger.warning(" ⚠️ No catalogues in DB to test detail endpoint") + except Exception as e: + logger.error(f" ❌ Error: {e}") + return False + + # Test 4: Get catalogue pages + logger.info("\n4. Testing GET /api/catalogues/{id}/pages") + try: + if cat: + response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}/pages") + if response.status_code == 200: + pages = response.json() + logger.info(f" ✅ Status 200 - Found {len(pages)} pages") + if pages: + logger.info(f" Sample page: {pages[0]['numero_page']} - {pages[0]['image_url'][:50]}...") + else: + logger.error(f" ❌ Status {response.status_code}") + return False + except Exception as e: + logger.error(f" ❌ Error: {e}") + return False + + # Test 5: Get stats (requires working DB) + logger.info("\n5. Testing GET /api/catalogues/admin/stats") + try: + response = await client.get(f"{API_BASE_URL}/catalogues/admin/stats") + if response.status_code == 200: + stats = response.json() + logger.info(f" ✅ Status 200 - Total catalogues: {stats['total_catalogues']}") + logger.info(f" Prochaine exécution: {stats['prochaine_execution']}") + else: + logger.error(f" ❌ Status {response.status_code}") + # Stats is optional, don't fail + except Exception as e: + logger.warning(f" ⚠️ Stats endpoint error (may require auth): {e}") + + logger.info("\n✅ All API endpoints are working!\n") + return True + + +async def main(): + """Run all verification tests.""" + logger.info("\n" + "=" * 60) + logger.info("BONIAL CATALOG MODULE - VERIFICATION SCRIPT") + logger.info("=" * 60 + "\n") + + results = [] + + # Phase 1: Database tables + results.append(("Database Tables", check_database_tables())) + + if not results[0][1]: + logger.error("\n❌ Database not ready. Please run: alembic upgrade head") + sys.exit(1) + + # Phase 2: Enseignes seeding + results.append(("Enseignes Seeding", check_enseignes_seeding())) + + # Phase 3: Scraper test (optional, can be slow) + scraper_test = input("\nRun scraper test? (Gifi - takes ~60s) [y/N]: ").lower() == "y" + if scraper_test: + results.append(("Scraper Test", await test_scraper())) + + # Phase 4: API endpoints (requires app to be running) + api_test = input("\nTest API endpoints? (App must be running on :8555) [y/N]: ").lower() == "y" + if api_test: + results.append(("API Endpoints", await test_api_endpoints())) + + # Summary + logger.info("\n" + "=" * 60) + logger.info("VERIFICATION SUMMARY") + logger.info("=" * 60) + + for name, passed in results: + status = "✅ PASSED" if passed else "❌ FAILED" + logger.info(f"{status}: {name}") + + all_passed = all(result[1] for result in results) + + if all_passed: + logger.info("\n✅ ALL TESTS PASSED - Bonial module is ready!") + else: + logger.error("\n❌ SOME TESTS FAILED - Please review errors above") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main())