feat: Add initial application structure, core data models, and a new catalog module with Bonial scraping capabilities.

This commit is contained in:
Michael committed 2025-11-29 19:26:56 +01:00
1 parent 906c6b483d
commit 58e6be3f0d
17 files changed
+2483 -7

No files matched your search

+2 -1
View File
@@ -29,8 +29,9 @@
- **Historique visuel** : Conserve l'historique des captures d'écran avec les métadonnées IA - **Historique visuel** : Conserve l'historique des captures d'écran avec les métadonnées IA
- **Défilement intelligent** : Scroll automatique pour charger le contenu différé - **Défilement intelligent** : Scroll automatique pour charger le contenu différé
- **Notifications** : Support multi-canaux (Discord, Telegram, Email, etc.) via [Apprise](https://github.com/caronc/apprise) - **Notifications** : Support multi-canaux (Discord, Telegram, Email, etc.) via [Apprise](https://github.com/caronc/apprise)
- **Catalogues promotionnels** : Consultation des catalogues et prospectus discount (Gifi, Action, Centrakor, etc.) scrapés depuis Bonial.fr
- **Interface en français** : Application entièrement traduite en français - **Interface en français** : Application entièrement traduite en français
- **Mode sombre** : Interface moderne avec support du thème clair/sombre - **Mode sombre** : Interface moderne avec support du thème clair/foncé
- **Dockerisé** : Déploiement facile avec Docker Compose - **Dockerisé** : Déploiement facile avec Docker Compose
## Prérequis ## Prérequis
@@ -0,0 +1,103 @@
"""Add catalog module tables
Revision ID: c1d2e3f4g5h6
Revises: b2c3d4e5f6g7
Create Date: 2025-11-29 18:56:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "c1d2e3f4g5h6"
down_revision: str | None = "b2c3d4e5f6g7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create tables for the catalog module."""
# Create enseignes table
op.create_table(
"enseignes",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("nom", sa.String(), nullable=False),
sa.Column("slug_bonial", sa.String(), nullable=False, unique=True, index=True),
sa.Column("logo_url", sa.String(), nullable=True),
sa.Column("couleur", sa.String(), nullable=False),
sa.Column("site_url", sa.String(), nullable=True),
sa.Column("description", sa.String(), nullable=True),
sa.Column("is_active", sa.Boolean(), default=True),
sa.Column("ordre_affichage", sa.Integer(), default=0),
sa.Column("created_at", sa.DateTime(), nullable=True),
sa.Column("updated_at", sa.DateTime(), nullable=True),
)
# Create catalogues table
op.create_table(
"catalogues",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=False, index=True),
sa.Column("titre", sa.String(), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("date_debut", sa.DateTime(), nullable=False, index=True),
sa.Column("date_fin", sa.DateTime(), nullable=False, index=True),
sa.Column("image_couverture_url", sa.String(), nullable=False),
sa.Column("catalogue_url", sa.String(), nullable=False),
sa.Column("nombre_pages", sa.Integer(), default=0),
sa.Column("statut", sa.String(), default="actif", index=True),
sa.Column("content_hash", sa.String(64), nullable=False, unique=True, index=True),
sa.Column("metadonnees", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=True),
sa.Column("updated_at", sa.DateTime(), nullable=True),
)
# Create catalogue_pages table
op.create_table(
"catalogue_pages",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("catalogue_id", sa.Integer(), sa.ForeignKey("catalogues.id"), nullable=False, index=True),
sa.Column("numero_page", sa.Integer(), nullable=False),
sa.Column("image_url", sa.String(), nullable=False),
sa.Column("image_thumbnail_url", sa.String(), nullable=True),
sa.Column("largeur", sa.Integer(), nullable=True),
sa.Column("hauteur", sa.Integer(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=True),
)
# Create unique index on catalogue_id + numero_page
op.create_index(
"idx_catalogue_page_numero",
"catalogue_pages",
["catalogue_id", "numero_page"],
unique=True,
)
# Create scraping_logs table
op.create_table(
"scraping_logs",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("date_execution", sa.DateTime(), nullable=True, index=True),
sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=True),
sa.Column("statut", sa.String(), nullable=False),
sa.Column("catalogues_trouves", sa.Integer(), default=0),
sa.Column("catalogues_nouveaux", sa.Integer(), default=0),
sa.Column("catalogues_mis_a_jour", sa.Integer(), default=0),
sa.Column("duree_secondes", sa.Float(), nullable=True),
sa.Column("message_erreur", sa.Text(), nullable=True),
sa.Column("details", sa.Text(), nullable=True),
)
def downgrade() -> None:
"""Drop catalog module tables."""
op.drop_table("scraping_logs")
op.drop_index("idx_catalogue_page_numero", table_name="catalogue_pages")
op.drop_table("catalogue_pages")
op.drop_table("catalogues")
op.drop_table("enseignes")
+20 -3
View File
@@ -13,9 +13,10 @@ from sqlalchemy import text
from app.database import SessionLocal, engine from app.database import SessionLocal, engine
from app.limiter import limiter from app.limiter import limiter
from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues
from app.services.scheduler_service import scheduled_refresh, scheduler from app.services.scheduler_service import scheduled_refresh, scheduler
from app.services import auth_service, search_service from app.services import auth_service, search_service, seed_enseignes
from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler
# Configure logging # Configure logging
logging.basicConfig( logging.basicConfig(
@@ -119,6 +120,13 @@ async def lifespan(app: FastAPI):
logger.info("Utilisateur admin par défaut créé (admin/admin)") logger.info("Utilisateur admin par défaut créé (admin/admin)")
else: else:
logger.info("Utilisateurs déjà initialisés") logger.info("Utilisateurs déjà initialisés")
# Seed enseignes for catalog module
created_enseignes = seed_enseignes.seed_enseignes(db)
if created_enseignes > 0:
logger.info(f"{created_enseignes} enseigne(s) de catalogues créée(s)")
else:
logger.info("Enseignes déjà initialisés")
finally: finally:
db.close() db.close()
except Exception as e: except Exception as e:
@@ -127,10 +135,16 @@ async def lifespan(app: FastAPI):
logger.info("Starting smart scheduler (Heartbeat: 1 minute)") logger.info("Starting smart scheduler (Heartbeat: 1 minute)")
scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True) scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True)
scheduler.start() scheduler.start()
# Start catalog scraping scheduler
logger.info("Starting catalog scraping scheduler (6h and 18h daily)")
start_catalog_scheduler()
logger.info("Application started") logger.info("Application started")
yield yield
logger.info("Shutting down scheduler...") logger.info("Shutting down schedulers...")
scheduler.shutdown(wait=True) scheduler.shutdown(wait=True)
stop_catalog_scheduler()
logger.info("Application shutdown complete") logger.info("Application shutdown complete")
@@ -170,6 +184,9 @@ for router in [notifications.router, items.router, settings.router, jobs.router,
app.include_router(search.router) app.include_router(search.router)
app.include_router(search_sites.router) app.include_router(search_sites.router)
# Catalog router (already has /api prefix)
app.include_router(catalogues.router)
# Auth router (already has /api prefix) # Auth router (already has /api prefix)
app.include_router(auth.router) app.include_router(auth.router)
+84
View File
@@ -113,3 +113,87 @@ class NotificationChannel(Base):
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
items = relationship("Item", back_populates="notification_channel") items = relationship("Item", back_populates="notification_channel")
# === CATALOG MODULE MODELS ===
class Enseigne(Base):
"""Stores/brands for promotional catalogs"""
__tablename__ = "enseignes"
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
nom: str = Column(String, nullable=False) # type: ignore # "Gifi"
slug_bonial: str = Column(String, unique=True, nullable=False, index=True) # type: ignore # "Gifi"
logo_url: str | None = Column(String, nullable=True) # type: ignore
couleur: str = Column(String, nullable=False) # type: ignore # "#E30613"
site_url: str | None = Column(String, nullable=True) # type: ignore
description: str | None = Column(String, nullable=True) # type: ignore
is_active: bool = Column(Boolean, default=True) # type: ignore
ordre_affichage: int = Column(Integer, default=0) # type: ignore
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore
# Relationships
catalogues = relationship("Catalogue", back_populates="enseigne", cascade="all, delete-orphan")
scraping_logs = relationship("ScrapingLog", back_populates="enseigne")
class Catalogue(Base):
"""Promotional catalogs from stores"""
__tablename__ = "catalogues"
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
enseigne_id: int = Column(Integer, ForeignKey("enseignes.id"), nullable=False, index=True) # type: ignore
titre: str = Column(String, nullable=False) # type: ignore
description: str | None = Column(Text, nullable=True) # type: ignore
date_debut: datetime = Column(DateTime, nullable=False, index=True) # type: ignore
date_fin: datetime = Column(DateTime, nullable=False, index=True) # type: ignore
image_couverture_url: str = Column(String, nullable=False) # type: ignore
catalogue_url: str = Column(String, nullable=False) # type: ignore # URL Bonial viewer
nombre_pages: int = Column(Integer, default=0) # type: ignore
statut: str = Column(String, default="actif", index=True) # type: ignore # actif, termine, erreur
content_hash: str = Column(String(64), unique=True, nullable=False, index=True) # type: ignore # SHA256
metadonnees: str | None = Column(Text, nullable=True) # type: ignore # JSON for additional data
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore
# Relationships
enseigne = relationship("Enseigne", back_populates="catalogues")
pages = relationship("CataloguePage", back_populates="catalogue", cascade="all, delete-orphan")
class CataloguePage(Base):
"""Individual pages of a catalog"""
__tablename__ = "catalogue_pages"
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
catalogue_id: int = Column(Integer, ForeignKey("catalogues.id"), nullable=False, index=True) # type: ignore
numero_page: int = Column(Integer, nullable=False) # type: ignore
image_url: str = Column(String, nullable=False) # type: ignore
image_thumbnail_url: str | None = Column(String, nullable=True) # type: ignore
largeur: int | None = Column(Integer, nullable=True) # type: ignore
hauteur: int | None = Column(Integer, nullable=True) # type: ignore
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
# Relationship
catalogue = relationship("Catalogue", back_populates="pages")
class ScrapingLog(Base):
"""Logs of scraping executions"""
__tablename__ = "scraping_logs"
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
date_execution: datetime = Column(DateTime, default=lambda: datetime.now(UTC), index=True) # type: ignore
enseigne_id: int | None = Column(Integer, ForeignKey("enseignes.id"), nullable=True) # type: ignore # NULL = all stores
statut: str = Column(String, nullable=False) # type: ignore # success, error, partial
catalogues_trouves: int = Column(Integer, default=0) # type: ignore
catalogues_nouveaux: int = Column(Integer, default=0) # type: ignore
catalogues_mis_a_jour: int = Column(Integer, default=0) # type: ignore
duree_secondes: float | None = Column(Float, nullable=True) # type: ignore
message_erreur: str | None = Column(Text, nullable=True) # type: ignore
details: str | None = Column(Text, nullable=True) # type: ignore # JSON
# Relationship
enseigne = relationship("Enseigne", back_populates="scraping_logs")
+317
View File
@@ -0,0 +1,317 @@
"""
API Router for catalog module
Endpoints for enseignes, catalogues, and admin operations
"""
import logging
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from sqlalchemy import func, and_
from app.database import get_db
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
from app.schemas_catalogues import (
EnseigneResponse,
CatalogueListResponse,
CatalogueDetailResponse,
CataloguePageResponse,
CataloguesPaginatedResponse,
PaginationMeta,
ScrapingLogResponse,
ScrapingStatsResponse,
)
from app.services.bonial_scraper import scrape_all_enseignes, scrape_enseigne
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/catalogues", tags=["catalogues"])
# === Enseignes Endpoints ===
@router.get("/enseignes", response_model=list[EnseigneResponse])
async def get_enseignes(
db: Session = Depends(get_db),
):
"""Get all active enseignes with catalog counts"""
enseignes = (
db.query(Enseigne)
.filter(Enseigne.is_active == True)
.order_by(Enseigne.ordre_affichage)
.all()
)
# Add catalog counts
result = []
for enseigne in enseignes:
count = (
db.query(func.count(Catalogue.id))
.filter(
and_(
Catalogue.enseigne_id == enseigne.id,
Catalogue.statut == "actif"
)
)
.scalar()
)
enseigne_data = EnseigneResponse.model_validate(enseigne)
enseigne_data.catalogues_actifs_count = count
result.append(enseigne_data)
return result
# === Catalogues Endpoints ===
@router.get("", response_model=CataloguesPaginatedResponse)
async def get_catalogues(
enseigne_ids: str | None = Query(None, description="Comma-separated enseigne IDs"),
statut: str | None = Query(None, description="Filter by status: actif, termine, erreur, tous"),
date_debut_min: datetime | None = Query(None, description="Start date minimum"),
date_fin_max: datetime | None = Query(None, description="End date maximum"),
recherche: str | None = Query(None, description="Search in titles"),
page: int = Query(1, ge=1, description="Page number"),
limit: int = Query(20, ge=1, le=100, description="Items per page"),
sort: str = Query("date_debut", description="Sort by: date_debut, date_fin, enseigne"),
order: str = Query("desc", description="Order: asc, desc"),
db: Session = Depends(get_db),
):
"""Get catalogues with filters and pagination"""
# Build query
query = db.query(Catalogue)
# Filter by enseignes
if enseigne_ids:
try:
ids = [int(id_str.strip()) for id_str in enseigne_ids.split(",")]
query = query.filter(Catalogue.enseigne_id.in_(ids))
except ValueError:
raise HTTPException(status_code=400, detail="Invalid enseigne_ids format")
# Filter by status
if statut and statut != "tous":
query = query.filter(Catalogue.statut == statut)
# Filter by dates
if date_debut_min:
query = query.filter(Catalogue.date_debut >= date_debut_min)
if date_fin_max:
query = query.filter(Catalogue.date_fin <= date_fin_max)
# Search in titles
if recherche:
query = query.filter(Catalogue.titre.ilike(f"%{recherche}%"))
# Sort
sort_column = {
"date_debut": Catalogue.date_debut,
"date_fin": Catalogue.date_fin,
"enseigne": Catalogue.enseigne_id,
}.get(sort, Catalogue.date_debut)
if order == "desc":
query = query.order_by(sort_column.desc())
else:
query = query.order_by(sort_column.asc())
# Count total
total = query.count()
# Paginate
offset = (page - 1) * limit
catalogues = query.offset(offset).limit(limit).all()
# Build response
data = []
for catalogue in catalogues:
cat_response = CatalogueListResponse.model_validate(catalogue)
data.append(cat_response)
pagination = PaginationMeta(
total=total,
page=page,
limit=limit,
pages_total=(total + limit - 1) // limit,
)
# Get last scraping time
last_scraping = (
db.query(ScrapingLog)
.order_by(ScrapingLog.date_execution.desc())
.first()
)
metadata = {
"derniere_mise_a_jour": last_scraping.date_execution if last_scraping else None
}
return CataloguesPaginatedResponse(
data=data,
pagination=pagination,
metadata=metadata,
)
@router.get("/{catalogue_id}", response_model=CatalogueDetailResponse)
async def get_catalogue_detail(
catalogue_id: int,
db: Session = Depends(get_db),
):
"""Get detailed information for a specific catalogue"""
catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
if not catalogue:
raise HTTPException(status_code=404, detail="Catalogue not found")
return CatalogueDetailResponse.model_validate(catalogue)
@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse])
async def get_catalogue_pages(
catalogue_id: int,
db: Session = Depends(get_db),
):
"""Get all pages of a catalogue"""
# Verify catalogue exists
catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
if not catalogue:
raise HTTPException(status_code=404, detail="Catalogue not found")
# Get pages ordered by page number
pages = (
db.query(CataloguePage)
.filter(CataloguePage.catalogue_id == catalogue_id)
.order_by(CataloguePage.numero_page)
.all()
)
return [CataloguePageResponse.model_validate(page) for page in pages]
# === Admin Endpoints ===
@router.post("/admin/scraping/trigger", response_model=dict)
async def trigger_scraping(
enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"),
db: Session = Depends(get_db),
):
"""Manually trigger catalog scraping"""
if enseigne_id:
# Scrape single enseigne
enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first()
if not enseigne:
raise HTTPException(status_code=404, detail="Enseigne not found")
# Run scraping asynchronously (in background)
import asyncio
from app.services.bonial_scraper import scrape_enseigne as scrape_ens
log = await scrape_ens(enseigne, db)
return {
"success": True,
"message": f"Scraping complete for {enseigne.nom}",
"catalogues_trouves": log.catalogues_trouves,
"catalogues_nouveaux": log.catalogues_nouveaux,
}
else:
# Scrape all enseignes
logs = await scrape_all_enseignes(db)
total_new = sum(log.catalogues_nouveaux for log in logs)
total_found = sum(log.catalogues_trouves for log in logs)
return {
"success": True,
"message": f"Scraping complete for all enseignes",
"enseignes_processed": len(logs),
"catalogues_trouves": total_found,
"catalogues_nouveaux": total_new,
}
@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse])
async def get_scraping_logs(
enseigne_id: int | None = Query(None, description="Filter by enseigne"),
statut: str | None = Query(None, description="Filter by status"),
limit: int = Query(50, ge=1, le=200, description="Number of logs to return"),
db: Session = Depends(get_db),
):
"""Get scraping execution logs"""
query = db.query(ScrapingLog)
if enseigne_id:
query = query.filter(ScrapingLog.enseigne_id == enseigne_id)
if statut:
query = query.filter(ScrapingLog.statut == statut)
logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all()
# Add enseigne name
result = []
for log in logs:
log_dict = ScrapingLogResponse.model_validate(log).model_dump()
if log.enseigne_id:
enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first()
log_dict["enseigne_nom"] = enseigne.nom if enseigne else None
else:
log_dict["enseigne_nom"] = "Toutes"
result.append(ScrapingLogResponse(**log_dict))
return result
@router.get("/admin/stats", response_model=ScrapingStatsResponse)
async def get_scraping_stats(
db: Session = Depends(get_db),
):
"""Get scraping statistics"""
# Total catalogues
total_catalogues = db.query(func.count(Catalogue.id)).scalar()
# Catalogues par enseigne
catalogues_par_enseigne = {}
enseignes = db.query(Enseigne).all()
for enseigne in enseignes:
count = (
db.query(func.count(Catalogue.id))
.filter(Catalogue.enseigne_id == enseigne.id)
.scalar()
)
catalogues_par_enseigne[enseigne.nom] = count
# Last scraping time
last_log = (
db.query(ScrapingLog)
.order_by(ScrapingLog.date_execution.desc())
.first()
)
derniere_mise_a_jour = last_log.date_execution if last_log else None
# Next execution (6:00 or 18:00)
now = datetime.now()
if now.hour < 6:
prochaine = f"Aujourd'hui à 6:00"
elif now.hour < 18:
prochaine = f"Aujourd'hui à 18:00"
else:
prochaine = f"Demain à 6:00"
return ScrapingStatsResponse(
total_catalogues=total_catalogues,
catalogues_par_enseigne=catalogues_par_enseigne,
derniere_mise_a_jour=derniere_mise_a_jour,
prochaine_execution=prochaine,
)
+109
View File
@@ -0,0 +1,109 @@
"""
Pydantic schemas for catalog module API
"""
from datetime import datetime
from pydantic import BaseModel
# === Enseigne Schemas ===
class EnseigneResponse(BaseModel):
id: int
nom: str
slug_bonial: str
logo_url: str | None
couleur: str
site_url: str | None
description: str | None
is_active: bool
ordre_affichage: int
# Statistics
catalogues_actifs_count: int | None = None
model_config = {"from_attributes": True}
# === Catalogue Schemas ===
class CatalogueListResponse(BaseModel):
id: int
enseigne: EnseigneResponse
titre: str
date_debut: datetime
date_fin: datetime
image_couverture_url: str
statut: str
nombre_pages: int
created_at: datetime
model_config = {"from_attributes": True}
class CatalogueDetailResponse(BaseModel):
id: int
enseigne: EnseigneResponse
titre: str
description: str | None
date_debut: datetime
date_fin: datetime
image_couverture_url: str
catalogue_url: str
statut: str
nombre_pages: int
metadonnees: str | None
created_at: datetime
updated_at: datetime
model_config = {"from_attributes": True}
class CataloguePageResponse(BaseModel):
id: int
numero_page: int
image_url: str
image_thumbnail_url: str | None
largeur: int | None
hauteur: int | None
model_config = {"from_attributes": True}
# === Pagination ===
class PaginationMeta(BaseModel):
total: int
page: int
limit: int
pages_total: int
class CataloguesPaginatedResponse(BaseModel):
data: list[CatalogueListResponse]
pagination: PaginationMeta
metadata: dict
# === Admin Schemas ===
class ScrapingLogResponse(BaseModel):
id: int
date_execution: datetime
enseigne_id: int | None
enseigne_nom: str | None
statut: str
catalogues_trouves: int
catalogues_nouveaux: int
catalogues_mis_a_jour: int
duree_secondes: float | None
message_erreur: str | None
model_config = {"from_attributes": True}
class ScrapingStatsResponse(BaseModel):
total_catalogues: int
catalogues_par_enseigne: dict[str, int]
derniere_mise_a_jour: datetime | None
prochaine_execution: str | None
+400
View File
@@ -0,0 +1,400 @@
"""
Bonial.fr Scraper Service
Scrapes promotional catalogs from Bonial.fr for configured enseignes.
Uses Playwright for browser automation.
"""
import asyncio
import hashlib
import logging
import re
from datetime import datetime
from typing import Any
from playwright.async_api import async_playwright, Page, Browser
from sqlalchemy.orm import Session
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
logger = logging.getLogger(__name__)
# Bonial selectors based on manual analysis
BONIAL_SELECTORS = {
"catalog_card": "section:has(img[src*='content-media.bonial.biz'])",
"catalog_title": "div.text-dark.text-lg.font-bold, h3, h2",
"catalog_image": "img[src*='content-media.bonial.biz']",
"catalog_link": "a:has-text('Ouvrir le catalogue')",
"cookie_accept": "button:has-text('OK'), button:has-text('Accepter')",
}
# Date pattern: "mar. 25/11 - lun. 08/12/2025"
DATE_PATTERN = r"(\d{2}/\d{2})\s*-\s*(\d{2}/\d{2}/\d{4})"
def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
"""
Parse Bonial date format to datetime objects.
Args:
date_str: Date string like "mar. 25/11 - lun. 08/12/2025"
Returns:
Tuple of (date_debut, date_fin)
Raises:
ValueError: If date format is invalid
"""
match = re.search(DATE_PATTERN, date_str)
if not match:
raise ValueError(f"Invalid date format: {date_str}")
debut_str, fin_str = match.groups()
# Extract year from end date
year = fin_str.split("/")[2]
# Add year to start date
debut_full = f"{debut_str}/{year}"
# Parse dates
debut = datetime.strptime(debut_full, "%d/%m/%Y")
fin = datetime.strptime(fin_str, "%d/%m/%Y")
return debut, fin
def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str:
"""
Compute SHA256 hash for duplicate detection.
Args:
enseigne_id: ID of the enseigne
titre: Catalog title
date_debut: Start date
Returns:
SHA256 hash string
"""
content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}"
return hashlib.sha256(content.encode()).hexdigest()
async def accept_cookies(page: Page) -> None:
"""Accept cookie consent banner if present."""
try:
await page.click(BONIAL_SELECTORS["cookie_accept"], timeout=3000)
logger.debug("Accepted cookies")
await asyncio.sleep(1)
except Exception:
logger.debug("No cookie banner found or already accepted")
async def scrape_catalog_list(
page: Page,
enseigne: Enseigne
) -> list[dict[str, Any]]:
"""
Scrape the list of catalogs for an ens enseigne from Bonial.
Args:
page: Playwright page object
enseigne: Enseigne object
Returns:
List of catalog data dictionaries
"""
url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}"
logger.info(f"Scraping catalog list for {enseigne.nom} from {url}")
await page.goto(url, wait_until="networkidle")
await accept_cookies(page)
# Wait for catalog cards to load
try:
await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000)
except Exception as e:
logger.error(f"No catalog cards found for {enseigne.nom}: {e}")
return []
# Extract all catalog cards
cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"])
logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}")
catalogues = []
for idx, card in enumerate(cards):
try:
# Extract title
title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"])
titre = await title_el.inner_text() if title_el else None
# Extract dates - look for any element containing date pattern
dates_text = await card.inner_text()
date_match = re.search(DATE_PATTERN, dates_text)
# Extract image
image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"])
image_url = await image_el.get_attribute("src") if image_el else None
# Extract link - try multiple strategies
link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"])
if not link_el:
# Fallback: click on image
link_el = image_el
catalogue_url = await link_el.get_attribute("href") if link_el else None
# Make URL absolute if relative
if catalogue_url and not catalogue_url.startswith("http"):
catalogue_url = f"https://www.bonial.fr{catalogue_url}"
if all([titre, date_match, image_url, catalogue_url]):
# Parse dates
try:
date_debut, date_fin = parse_bonial_dates(dates_text)
except ValueError as e:
logger.warning(f"Failed to parse dates for catalog {idx}: {e}")
continue
catalogues.append({
"titre": titre.strip(),
"date_debut": date_debut,
"date_fin": date_fin,
"image_couverture_url": image_url,
"catalogue_url": catalogue_url,
})
logger.debug(f"Extracted catalog: {titre}")
else:
logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={bool(date_match)}, image={bool(image_url)}, url={bool(catalogue_url)}")
except Exception as e:
logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}")
continue
logger.info(f"Successfully extracted {len(catalogues)} catalogs for {enseigne.nom}")
return catalogues
async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]:
"""
Scrape all pages of a catalog from the Bonial viewer.
Args:
page: Playwright page object
catalogue_url: URL of the catalog viewer
Returns:
List of page data dictionaries
"""
logger.info(f"Scraping catalog pages from {catalogue_url}")
await page.goto(catalogue_url, wait_until="networkidle")
await asyncio.sleep(2) # Wait for viewer to initialize
# Try to close any popup/modal
try:
await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000)
await asyncio.sleep(1)
except Exception:
pass
# Find all images from content-media.bonial.biz
images = await page.query_selector_all("img[src*='content-media.bonial.biz']")
if not images:
logger.warning(f"No images found in catalog viewer: {catalogue_url}")
return []
# Extract unique image URLs (filter duplicates and thumbnails)
seen_urls = set()
pages = []
for idx, img in enumerate(images):
try:
src = await img.get_attribute("src")
if src and src not in seen_urls:
# Filter out blurred/low-res images if possible
# Full res images are typically larger
width = await img.evaluate("el => el.naturalWidth")
height = await img.evaluate("el => el.naturalHeight")
# Skip very small images (likely thumbnails)
if width < 200 or height < 200:
continue
seen_urls.add(src)
pages.append({
"numero_page": len(pages) + 1,
"image_url": src,
"largeur": width,
"hauteur": height,
})
except Exception as e:
logger.debug(f"Error processing image {idx}: {e}")
continue
logger.info(f"Found {len(pages)} pages in catalog")
return pages
async def scrape_enseigne(
enseigne: Enseigne,
db: Session,
browser: Browser | None = None
) -> ScrapingLog:
"""
Scrape all catalogs for a specific enseigne.
Args:
enseigne: Enseigne to scrape
db: Database session
browser: Optional existing browser instance
Returns:
ScrapingLog object with execution details
"""
start_time = datetime.now()
log = ScrapingLog(
enseigne_id=enseigne.id,
statut="error", # Will be updated
catalogues_trouves=0,
catalogues_nouveaux=0,
catalogues_mis_a_jour=0,
)
own_browser = browser is None
try:
# Create browser if not provided
if own_browser:
p = await async_playwright().start()
browser = await p.chromium.launch(headless=True)
page = await browser.new_page()
# Set realistic user agent
await page.set_extra_http_headers({
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
})
# Scrape catalog list
catalogues_data = await scrape_catalog_list(page, enseigne)
log.catalogues_trouves = len(catalogues_data)
if not catalogues_data:
log.statut = "success"
log.message_erreur = "No catalogs found (may be normal)"
await page.close()
return log
# Process each catalog
for cat_data in catalogues_data:
try:
# Compute hash for duplicate detection
content_hash = compute_catalog_hash(
enseigne.id,
cat_data["titre"],
cat_data["date_debut"]
)
# Check if catalog already exists
existing = db.query(Catalogue).filter_by(content_hash=content_hash).first()
if existing:
logger.debug(f"Catalog already exists: {cat_data['titre']}")
continue
# Scrape catalog pages
await asyncio.sleep(3) # Respectful delay
pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"])
# Create new catalog
catalogue = Catalogue(
enseigne_id=enseigne.id,
titre=cat_data["titre"],
date_debut=cat_data["date_debut"],
date_fin=cat_data["date_fin"],
image_couverture_url=cat_data["image_couverture_url"],
catalogue_url=cat_data["catalogue_url"],
nombre_pages=len(pages_data),
content_hash=content_hash,
statut="actif" if cat_data["date_fin"] >= datetime.now() else "termine",
)
db.add(catalogue)
db.flush() # Get catalogue.id
# Add pages
for page_data in pages_data:
cat_page = CataloguePage(
catalogue_id=catalogue.id,
**page_data
)
db.add(cat_page)
log.catalogues_nouveaux += 1
logger.info(f"Created catalog: {cat_data['titre']} ({len(pages_data)} pages)")
except Exception as e:
logger.error(f"Error processing catalog {cat_data.get('titre')}: {e}")
continue
await page.close()
# Update status
log.statut = "success" if log.catalogues_nouveaux > 0 else "partial"
db.commit()
except Exception as e:
logger.error(f"Error scraping enseigne {enseigne.nom}: {e}")
log.statut = "error"
log.message_erreur = str(e)
db.rollback()
finally:
if own_browser and browser:
await browser.close()
# Record duration
log.duree_secondes = (datetime.now() - start_time).total_seconds()
db.add(log)
db.commit()
return log
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
"""
Scrape all active enseignes.
Args:
db: Database session
Returns:
List of ScrapingLog objects
"""
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
logs = []
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
for enseigne in enseignes:
try:
await asyncio.sleep(2) # Delay between enseignes
log = await scrape_enseigne(enseigne, db, browser)
logs.append(log)
except Exception as e:
logger.error(f"Failed to scrape {enseigne.nom}: {e}")
await browser.close()
logger.info(f"Scraping complete: {len(logs)} enseignes processed")
return logs
+62
View File
@@ -0,0 +1,62 @@
"""
Scheduler for automated catalog scraping
Runs scraping jobs twice daily (6h and 18h) for all active enseignes.
"""
import logging
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from app.database import SessionLocal
from app.services.bonial_scraper import scrape_all_enseignes
logger = logging.getLogger(__name__)
# Global scheduler instance
scheduler = AsyncIOScheduler()
async def scheduled_scrape_job():
"""Scheduled job to scrape all active enseignes"""
logger.info("Starting scheduled catalog scraping job")
db = SessionLocal()
try:
logs = await scrape_all_ense ignes(db)
total_new = sum(log.catalogues_nouveaux for log in logs)
total_found = sum(log.catalogues_trouves for log in logs)
logger.info(
f"Scheduled scraping complete: "
f"{len(logs)} enseignes processed, "
f"{total_found} catalogues found, "
f"{total_new} new catalogues created"
)
except Exception as e:
logger.error(f"Error in scheduled scraping job: {e}")
finally:
db.close()
def start_scheduler():
"""Start the scheduler with configured jobs"""
# Schedule scraping twice daily at 6:00 and 18:00
scheduler.add_job(
scheduled_scrape_job,
CronTrigger(hour="6,18", minute=0),
id="catalog_scraping",
name="Scrape Bonial catalogs",
replace_existing=True,
)
scheduler.start()
logger.info("Catalog scraping scheduler started (runs at 6:00 and 18:00)")
def stop_scheduler():
"""Stop the scheduler"""
if scheduler.running:
scheduler.shutdown()
logger.info("Catalog scraping scheduler stopped")
+128
View File
@@ -0,0 +1,128 @@
"""
Seed default enseignes (stores) for the catalog module
"""
import logging
from sqlalchemy.orm import Session
from app.models import Enseigne
logger = logging.getLogger(__name__)
# Data for the 9 enseignes
ENSEIGNES_DATA = [
{
"nom": "Gifi",
"slug_bonial": "Gifi",
"couleur": "#E30613",
"site_url": "https://www.gifi.fr",
"description": "Décoration, maison, bazar",
"ordre_affichage": 1,
},
{
"nom": "Action",
"slug_bonial": "Action",
"couleur": "#0066B3",
"site_url": "https://www.action.com/fr-fr/",
"description": "Discount non-alimentaire",
"ordre_affichage": 2,
},
{
"nom": "Centrakor",
"slug_bonial": "Centrakor",
"couleur": "#E94E1B",
"site_url": "https://www.centrakor.com",
"description": "Décoration, maison",
"ordre_affichage": 3,
},
{
"nom": "La Foir'Fouille",
"slug_bonial": "La-Foir-Fouille",
"couleur": "#009639",
"site_url": "https://www.lafoirfouille.fr",
"description": "Bazar, décoration",
"ordre_affichage": 4,
},
{
"nom": "Stokomani",
"slug_bonial": "Stokomani",
"couleur": "#FF6600",
"site_url": "https://www.stokomani.fr",
"description": "Déstockage textile et maison",
"ordre_affichage": 5,
},
{
"nom": "B&M",
"slug_bonial": "BM",
"couleur": "#D4145A",
"site_url": "https://bmstores.fr",
"description": "Discount britannique",
"ordre_affichage": 6,
},
{
"nom": "L'Incroyable",
"slug_bonial": "L-incroyable",
"couleur": "#8B0000",
"site_url": "https://www.lincroyable.fr",
"description": "Décoration et mobilier discount (Groupe Althys, siège à Denain)",
"ordre_affichage": 7,
},
{
"nom": "Bazarland",
"slug_bonial": "Bazarland",
"couleur": "#FFCC00",
"site_url": None,
"description": "Bazar discount",
"ordre_affichage": 8,
},
{
"nom": "Noz",
"slug_bonial": "Noz",
"couleur": "#003366",
"site_url": "https://www.noz.fr",
"description": "Déstockage généraliste",
"ordre_affichage": 9,
},
]
def seed_enseignes(db: Session) -> int:
"""
Seed the default enseignes if they don't exist.
Returns:
Number of enseignes created
"""
created = 0
for data in ENSEIGNES_DATA:
# Check if enseigne already exists
existing = db.query(Enseigne).filter_by(slug_bonial=data["slug_bonial"]).first()
if not existing:
enseigne = Enseigne(**data)
db.add(enseigne)
created += 1
logger.info(f"Created enseigne: {data['nom']}")
else:
logger.debug(f"Enseigne already exists: {data['nom']}")
db.commit()
logger.info(f"Seeding complete: {created} enseigne(s) created")
return created
def get_enseigne_by_slug(db: Session, slug: str) -> Enseigne | None:
"""Get an enseigne by its Bonial slug"""
return db.query(Enseigne).filter_by(slug_bonial=slug).first()
def get_all_active_enseignes(db: Session) -> list[Enseigne]:
"""Get all active enseignes ordered by display order"""
return (
db.query(Enseigne)
.filter_by(is_active=True)
.order_by(Enseigne.ordre_affichage)
.all()
)
+329
View File
@@ -0,0 +1,329 @@
# API Documentation - Module Catalogues Bonial
## Vue d'ensemble
Le module Catalogues permet de consulter les catalogues promotionnels des enseignes discount scrapers depuis Bonial.fr.
**Base URL**: `/api/catalogues`
## Endpoints Publics
### 1. Liste des Enseignes
`GET /api/catalogues/enseignes`
Retourne la liste de toutes les enseignes actives avec le nombre de catalogues en cours.
**Réponse** : `200 OK`
```json
[
{
"id": 1,
"nom": "Gifi",
"slug_bonial": "Gifi",
"logo_url": null,
"couleur": "#E30613",
"site_url": "https://www.gifi.fr",
"description": "Décoration, maison, bazar",
"is_active": true,
"ordre_affichage": 1,
"catalogues_actifs_count": 3
}
]
```
---
### 2. Liste des Catalogues (Paginée)
`GET /api/catalogues`
Retourne la liste paginée des catalogues avec filtres.
**Paramètres Query** :
- `page` (int, default=1): Numéro de page
- `limit` (int, default=20, max=100): Nombre d'éléments par page
- `enseigne_ids` (string, optional): IDs d'enseignes séparés par virgule (ex: `1,2,3`)
- `statut` (string, optional): Filtre par statut (`actif`, `termine`, `tous`)
- `date_debut_min` (datetime, optional): Date de début minimum
- `date_fin_max` (datetime, optional): Date de fin maximum
- `recherche` (string, optional): Recherche dans les titres
- `sort` (string, default=`date_debut`): Tri (`date_debut`, `date_fin`, `enseigne`)
- `order` (string, default=`desc`): Ordre (`asc`, `desc`)
**Réponse** : `200 OK`
```json
{
"data": [
{
"id": 42,
"enseigne": {
"id": 1,
"nom": "Gifi",
"couleur": "#E30613",
...
},
"titre": "Catalogue Gifi Noël 2024",
"date_debut": "2024-11-25T00:00:00",
"date_fin": "2024-12-24T23:59:59",
"image_couverture_url": "https://content-media.bonial.biz/...",
"statut": "actif",
"nombre_pages": 24,
"created_at": "2024-11-29T06:00:00"
}
],
"pagination": {
"total": 156,
"page": 1,
"limit": 20,
"pages_total": 8
},
"metadata": {
"derniere_mise_a_jour": "2024-11-29T06:15:32"
}
}
```
---
### 3. Détail d'un Catalogue
`GET /api/catalogues/{catalogue_id}`
Retourne les informations détaillées d'un catalogue.
**Paramètres Path** :
- `catalogue_id` (int): ID du catalogue
**Réponse** : `200 OK`
```json
{
"id": 42,
"enseigne": {
"id": 1,
"nom": "Gifi",
...
},
"titre": "Catalogue Gifi Noël 2024",
"description": null,
"date_debut": "2024-11-25T00:00:00",
"date_fin": "2024-12-24T23:59:59",
"image_couverture_url": "https://content-media.bonial.biz/...",
"catalogue_url": "https://www.bonial.fr/...",
"statut": "actif",
"nombre_pages": 24,
"metadonnees": null,
"created_at": "2024-11-29T06:00:00",
"updated_at": "2024-11-29T06:00:00"
}
```
**Erreurs** :
- `404 Not Found`: Catalogue introuvable
---
### 4. Pages d'un Catalogue
`GET /api/catalogues/{catalogue_id}/pages`
Retourne toutes les pages d'un catalogue, ordonnées par numéro.
**Paramètres Path** :
- `catalogue_id` (int): ID du catalogue
**Réponse** : `200 OK`
```json
[
{
"id": 128,
"numero_page": 1,
"image_url": "https://content-media.bonial.biz/.../page_1.jpg",
"image_thumbnail_url": null,
"largeur": 1200,
"hauteur": 1600
},
{
"id": 129,
"numero_page": 2,
"image_url": "https://content-media.bonial.biz/.../page_2.jpg",
"image_thumbnail_url": null,
"largeur": 1200,
"hauteur": 1600
}
]
```
**Erreurs** :
- `404 Not Found`: Catalogue introuvable
---
## Endpoints Admin (Authentification requise)
### 5. Déclencher Scraping Manuel
`POST /api/catalogues/admin/scraping/trigger`
Déclenche un scraping manuel des catalogues Bonial.
**Authentification** : Requise (JWT token + rôle `admin`)
**Paramètres Query** :
- `enseigne_id` (int, optional): ID de l'enseigne à scraper. Si omis, scrape toutes les enseignes actives.
**Réponse** : `200 OK`
```json
{
"success": true,
"message": "Scraping complete for Gifi",
"catalogues_trouves": 4,
"catalogues_nouveaux": 2
}
```
Ou pour toutes les enseignes :
```json
{
"success": true,
"message":" Scraping complete for all enseignes",
"enseignes_processed": 9,
"catalogues_trouves": 38,
"catalogues_nouveaux": 12
}
```
**Erreurs** :
- `403 Forbidden`: Accès admin requis
- `404 Not Found`: Enseigne introuvable
---
### 6. Historique des Scraping
`GET /api/catalogues/admin/scraping/logs`
Retourne l'historique des exécutions de scraping.
**Authentification** : Requise (JWT token + rôle `admin`)
**Paramètres Query** :
- `limit` (int, default=50, max=200): Nombre de logs à retourner
- `enseigne_id` (int, optional): Filtre par enseigne
- `statut` (string, optional): Filtre par statut (`success`, `error`, `partial`)
**Réponse** : `200 OK`
```json
[
{
"id": 15,
"date_execution": "2024-11-29T06:00:15",
"enseigne_id": 1,
"enseigne_nom": "Gifi",
"statut": "success",
"catalogues_trouves": 4,
"catalogues_nouveaux": 2,
"catalogues_mis_a_jour": 0,
"duree_secondes": 45.2,
"message_erreur": null
}
]
```
---
### 7. Statistiques de Scraping
`GET /api/catalogues/admin/stats`
Retourne les statistiques globales du module catalogues.
**Authentification** : Requise (JWT token + rôle `admin`)
**Réponse** : `200 OK`
```json
{
"total_catalogues": 156,
"catalogues_par_enseigne": {
"Gifi": 18,
"Action": 22,
"Centrakor": 15,
"La Foir'Fouille": 12,
"Stokomani": 19,
"B&M": 14,
"L'Incroyable": 11,
"Bazarland": 8,
"Noz": 17
},
"derniere_mise_a_jour": "2024-11-29T06:15:32",
"prochaine_execution": "Aujourd'hui à 18:00"
}
```
---
## Erreurs Communes
### 400 Bad Request
```json
{
"detail": "Invalid enseigne_ids format"
}
```
### 403 Forbidden
```json
{
"detail": "Admin access required"
}
```
### 404 Not Found
```json
{
"detail": "Catalogue not found"
}
```
### 500 Internal Server Error
```json
{
"detail": "Internal server error"
}
```
---
## Scheduler Automatique
Le scraping est exécuté automatiquement **2 fois par jour** :
- **6h00** : Scraping matinal
- **18h00** : Scraping du soir
Les catalogues sont automatiquement marqués comme `termine` lorsque leur `date_fin` est dépassée.
---
## Notes d'Implémentation
### Détection de Doublons
Le système utilise un hash SHA-256 basé sur :
- `enseigne_id`
- `titre`
- `date_debut`
Cela évite de créer des doublons même si le catalogue est re-scrapé.
### Performance
- Pagination obligatoire (max 100 items/page)
- Index DB sur `enseigne_id`, `date_debut`, `date_fin`, `statut`
- Scraping asynchrone (n'impacte pas les performances API)
### Formats de Date
Toutes les dates sont en **ISO 8601** avec timezone UTC :
```
2024-11-29T06:15:32+00:00
```
+2
View File
@@ -5,6 +5,7 @@ import Layout from '@/components/layout/Layout'
import Dashboard from '@/pages/Dashboard' import Dashboard from '@/pages/Dashboard'
import Search from '@/pages/Search' import Search from '@/pages/Search'
import MultiSearch from '@/pages/MultiSearch' import MultiSearch from '@/pages/MultiSearch'
import Catalogues from '@/pages/Catalogues'
import Login from '@/pages/Login' import Login from '@/pages/Login'
import Admin from '@/pages/Admin' import Admin from '@/pages/Admin'
import { useTheme } from '@/hooks/use-theme' import { useTheme } from '@/hooks/use-theme'
@@ -77,6 +78,7 @@ function AppRoutes() {
<Route path="/" element={<Dashboard />} /> <Route path="/" element={<Dashboard />} />
<Route path="/search" element={<Search />} /> <Route path="/search" element={<Search />} />
<Route path="/compare" element={<MultiSearch />} /> <Route path="/compare" element={<MultiSearch />} />
<Route path="/catalogues" element={<Catalogues />} />
<Route <Route
path="/admin" path="/admin"
element={ element={
@@ -0,0 +1,229 @@
import React, { useState, useEffect } from 'react';
import axios from 'axios';
import { toast } from 'sonner';
import { Loader2, Play, Clock, CheckCircle2, XCircle, AlertCircle } from 'lucide-react';
const API_BASE = '/api/catalogues/admin';
export default function CatalogueAdmin() {
const [loading, setLoading] = useState(false);
const [scraping, setScraping] = useState(false);
const [logs, setLogs] = useState([]);
const [stats, setStats] = useState(null);
const [enseignes, setEnseignes] = useState([]);
useEffect(() => {
loadEnseignes();
loadStats();
loadLogs();
}, []);
const loadEnseignes = async () => {
try {
const response = await axios.get('/api/catalogues/enseignes');
setEnseignes(response.data);
} catch (error) {
console.error('Error loading enseignes:', error);
}
};
const loadStats = async () => {
try {
const response = await axios.get(`${API_BASE}/stats`);
setStats(response.data);
} catch (error) {
console.error('Error loading stats:', error);
}
};
const loadLogs = async () => {
setLoading(true);
try {
const response = await axios.get(`${API_BASE}/scraping/logs?limit=20`);
setLogs(response.data);
} catch (error) {
console.error('Error loading logs:', error);
toast.error('Erreur lors du chargement des logs');
} finally {
setLoading(false);
}
};
const triggerScraping = async (enseigneId = null) => {
setScraping(true);
try {
const params = enseigneId ? { enseigne_id: enseigneId } : {};
const response = await axios.post(`${API_BASE}/scraping/trigger`, null, { params });
toast.success(response.data.message);
toast.info(`${response.data.catalogues_nouveaux} nouveaux catalogues trouvés`);
// Reload data
await Promise.all([loadStats(), loadLogs()]);
} catch (error) {
console.error('Error triggering scraping:', error);
toast.error('Erreur lors du déclenchement du scraping');
} finally {
setScraping(false);
}
};
const getStatusIcon = (status) => {
switch (status) {
case 'success':
return <CheckCircle2 className="h-5 w-5 text-green-500" />;
case 'error':
return <XCircle className="h-5 w-5 text-red-500" />;
case 'partial':
return <AlertCircle className="h-5 w-5 text-yellow-500" />;
default:
return <Clock className="h-5 w-5 text-gray-500" />;
}
};
const formatDate = (dateString) => {
const date = new Date(dateString);
return date.toLocaleString('fr-FR');
};
return (
<div className="space-y-6">
<div>
<h2 className="text-2xl font-bold mb-2">Administration Catalogues</h2>
<p className="text-muted-foreground">Gérer le scraping des catalogues Bonial</p>
</div>
{/* Stats Cards */}
{stats && (
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
<div className="bg-card rounded-lg border p-6">
<h3 className="text-sm font-medium text-muted-foreground mb-2">Total Catalogues</h3>
<p className="text-3xl font-bold">{stats.total_catalogues}</p>
</div>
<div className="bg-card rounded-lg border p-6">
<h3 className="text-sm font-medium text-muted-foreground mb-2">Dernière Mise à Jour</h3>
<p className="text-lg font-medium">
{stats.derniere_mise_a_jour
? formatDate(stats.derniere_mise_a_jour)
: 'Jamais'}
</p>
</div>
<div className="bg-card rounded-lg border p-6">
<h3 className="text-sm font-medium text-muted-foreground mb-2">Prochaine Exécution</h3>
<p className="text-lg font-medium">{stats.prochaine_execution}</p>
</div>
</div>
)}
{/* Manual Scraping */}
<div className="bg-card rounded-lg border p-6">
<h3 className="text-lg font-semibold mb-4">Scraping Manuel</h3>
<div className="space-y-4">
<button
onClick={() => triggerScraping()}
disabled={scraping}
className="w-full md:w-auto px-6 py-3 bg-primary text-primary-foreground rounded-lg hover:bg-primary/90 disabled:opacity-50 flex items-center gap-2"
>
{scraping ? (
<>
<Loader2 className="h-5 w-5 animate-spin" />
Scraping en cours...
</>
) : (
<>
<Play className="h-5 w-5" />
Scraper toutes les enseignes
</>
)}
</button>
<div className="border-t pt-4">
<p className="text-sm text-muted-foreground mb-3">Ou scraper une enseigne spécifique:</p>
<div className="grid grid-cols-2 md:grid-cols-3 gap-2">
{enseignes.map(enseigne => (
<button
key={enseigne.id}
onClick={() => triggerScraping(enseigne.id)}
disabled={scraping}
className="px-4 py-2 rounded-lg border hover:bg-secondary disabled:opacity-50 text-left"
>
<div className="font-medium">{enseigne.nom}</div>
<div className="text-xs text-muted-foreground">
{enseigne.catalogues_actifs_count} catalogue(s)
</div>
</button>
))}
</div>
</div>
</div>
</div>
{/* Catalogues par Enseigne */}
{stats && (
<div className="bg-card rounded-lg border p-6">
<h3 className="text-lg font-semibold mb-4">Catalogues par Enseigne</h3>
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
{Object.entries(stats.catalogues_par_enseigne).map(([nom, count]) => (
<div key={nom} className="flex justify-between items-center p-3 rounded-lg bg-secondary">
<span className="font-medium">{nom}</span>
<span className="text-lg font-bold text-primary">{count}</span>
</div>
))}
</div>
</div>
)}
{/* Scraping Logs */}
<div className="bg-card rounded-lg border p-6">
<div className="flex justify-between items-center mb-4">
<h3 className="text-lg font-semibold">Historique des Scraping</h3>
<button
onClick={loadLogs}
disabled={loading}
className="px-3 py-1 text-sm rounded-lg border hover:bg-secondary"
>
Actualiser
</button>
</div>
{loading ? (
<div className="flex justify-center py-8">
<Loader2 className="h-6 w-6 animate-spin text-primary" />
</div>
) : logs.length === 0 ? (
<p className="text-center text-muted-foreground py-8">Aucun log disponible</p>
) : (
<div className="space-y-2">
{logs.map(log => (
<div
key={log.id}
className="flex items-start gap-3 p-4 rounded-lg border hover:bg-secondary/50 transition-colors"
>
<div className="mt-0.5">
{getStatusIcon(log.statut)}
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 mb-1">
<span className="font-medium">{log.enseigne_nom || 'Toutes'}</span>
<span className="text-xs text-muted-foreground">
{formatDate(log.date_execution)}
</span>
</div>
<div className="text-sm text-muted-foreground">
{log.catalogues_trouves} trouvé(s) • {log.catalogues_nouveaux} nouveau(x)
{log.duree_secondes && ` • ${log.duree_secondes.toFixed(1)}s`}
</div>
{log.message_erreur && (
<div className="text-sm text-red-500 mt-1">
{log.message_erreur}
</div>
)}
</div>
</div>
))}
</div>
)}
</div>
</div>
);
}
+2 -1
View File
@@ -1,7 +1,7 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { Link, useLocation, useNavigate } from 'react-router-dom'; import { Link, useLocation, useNavigate } from 'react-router-dom';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { LayoutDashboard, Search, TrendingUp, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react'; import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet'; import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
@@ -23,6 +23,7 @@ const Layout = ({ children, theme, toggleTheme }) => {
{ icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' }, { icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' },
{ icon: Search, label: t('nav.search') || 'Recherche', path: '/search' }, { icon: Search, label: t('nav.search') || 'Recherche', path: '/search' },
{ icon: TrendingUp, label: 'Comparateur', path: '/compare' }, { icon: TrendingUp, label: 'Comparateur', path: '/compare' },
{ icon: BookOpen, label: 'Catalogues', path: '/catalogues' },
]; ];
// Add admin link for admin users // Add admin link for admin users
+7 -2
View File
@@ -7,7 +7,7 @@ import {
Settings as SettingsIcon, Cpu, Clock, Bell, Globe, Settings as SettingsIcon, Cpu, Clock, Bell, Globe,
ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2, ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2,
Sparkles, Eye, Code, Brain, DollarSign, Filter, Sparkles, Eye, Code, Brain, DollarSign, Filter,
TrendingDown, CheckCircle2, Package, Bug, Download TrendingDown, CheckCircle2, Package, Bug, Download, BookOpen
} from 'lucide-react'; } from 'lucide-react';
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
import { Label } from '@/components/ui/label'; import { Label } from '@/components/ui/label';
@@ -18,6 +18,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@
import { Slider } from '@/components/ui/slider'; import { Slider } from '@/components/ui/slider';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { useAuth } from '@/hooks/use-auth'; import { useAuth } from '@/hooks/use-auth';
import CatalogueAdmin from '@/components/dashboard/CatalogueAdmin';
const API_URL = '/api'; const API_URL = '/api';
@@ -495,6 +496,7 @@ export default function Admin() {
{ id: 'sites', label: 'Sites', icon: Globe }, { id: 'sites', label: 'Sites', icon: Globe },
{ id: 'jobs', label: 'Jobs', icon: Clock }, { id: 'jobs', label: 'Jobs', icon: Clock },
{ id: 'notifications', label: 'Notifications', icon: Bell }, { id: 'notifications', label: 'Notifications', icon: Bell },
{ id: 'catalogues', label: 'Catalogues', icon: BookOpen },
{ id: 'debug', label: 'Debug', icon: Bug }, { id: 'debug', label: 'Debug', icon: Bug },
]; ];
@@ -1120,6 +1122,9 @@ export default function Admin() {
</CardContent> </CardContent>
</Card> </Card>
)} )}
</div >
{/* Catalogues Tab */}
{activeTab === 'catalogues' && <CatalogueAdmin />}
</div>
); );
} }
+323
View File
@@ -0,0 +1,323 @@
import React, { useEffect, useState } from 'react';
import { toast } from 'sonner';
import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye } from 'lucide-react';
import axios from 'axios';
const API_BASE = '/api/catalogues';
export default function Catalogues() {
const [loading, setLoading] = useState(true);
const [enseignes, setEnseignes] = useState([]);
const [catalogues, setCatalogues] = useState([]);
const [selectedEnseigne, setSelectedEnseigne] = useState(null);
const [pagination, setPagination] = useState({ page: 1, limit: 12, total: 0 });
const [selectedCatalogue, setSelectedCatalogue] = useState(null);
const [cataloguePages, setCataloguePages] = useState([]);
const [searchQuery, setSearchQuery] = useState('');
useEffect(() => {
loadEnseignes();
loadCatalogues();
}, []);
useEffect(() => {
loadCatalogues();
}, [pagination.page, selectedEnseigne, searchQuery]);
const loadEnseignes = async () => {
try {
const response = await axios.get(`${API_BASE}/enseignes`);
setEnseignes(response.data);
} catch (error) {
console.error('Error loading enseignes:', error);
toast.error('Erreur lors du chargement des enseignes');
}
};
const loadCatalogues = async () => {
setLoading(true);
try {
const params = {
page: pagination.page,
limit: pagination.limit,
statut: 'actif',
};
if (selectedEnseigne) {
params.enseigne_ids = selectedEnseigne.id.toString();
}
if (searchQuery) {
params.recherche = searchQuery;
}
const response = await axios.get(API_BASE, { params });
setCatalogues(response.data.data);
setPagination(prev => ({ ...prev, total: response.data.pagination.total }));
} catch (error) {
console.error('Error loading catalogues:', error);
toast.error('Erreur lors du chargement des catalogues');
} finally {
setLoading(false);
}
};
const loadCataloguePages = async (catalogueId) => {
try {
const response = await axios.get(`${API_BASE}/${catalogueId}/pages`);
setCataloguePages(response.data);
} catch (error) {
console.error('Error loading pages:', error);
toast.error('Erreur lors du chargement des pages');
}
};
const openCatalogue = (catalogue) => {
setSelectedCatalogue(catalogue);
loadCataloguePages(catalogue.id);
};
const closeCatalogue = () => {
setSelectedCatalogue(null);
setCataloguePages([]);
};
const formatDate = (dateString) => {
const date = new Date(dateString);
return date.toLocaleDateString('fr-FR', { day: 'numeric', month: 'short', year: 'numeric' });
};
const isValidNow = (catalogue) => {
const now = new Date();
const debut = new Date(catalogue.date_debut);
const fin = new Date(catalogue.date_fin);
return now >= debut && now <= fin;
};
return (
<div className="p-6 max-w-7xl mx-auto">
{/* Header */}
<div className="mb-8">
<h1 className="text-3xl font-bold mb-2">Catalogues Promotionnels</h1>
<p className="text-muted-foreground">
Consultez les catalogues et prospectus des enseignes discount
</p>
</div>
{/* Search Bar */}
<div className="mb-6">
<div className="relative">
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
<input
type="text"
placeholder="Rechercher dans les catalogues..."
className="w-full pl-10 pr-4 py-3 rounded-lg border bg-background"
value={searchQuery}
onChange={(e) => {
setSearchQuery(e.target.value);
setPagination(prev => ({ ...prev, page: 1 }));
}}
/>
</div>
</div>
{/* Enseignes Filter */}
<div className="mb-6">
<div className="flex gap-2 overflow-x-auto pb-2">
<button
onClick={() => {
setSelectedEnseigne(null);
setPagination(prev => ({ ...prev, page: 1 }));
}}
className={`px-4 py-2 rounded-lg whitespace-nowrap transition-colors ${!selectedEnseigne
? 'bg-primary text-primary-foreground'
: 'bg-secondary hover:bg-secondary/80'
}`}
>
Toutes les enseignes
</button>
{enseignes.map(enseigne => (
<button
key={enseigne.id}
onClick={() => {
setSelectedEnseigne(enseigne);
setPagination(prev => ({ ...prev, page: 1 }));
}}
className={`px-4 py-2 rounded-lg whitespace-nowrap transition-colors ${selectedEnseigne?.id === enseigne.id
? 'bg-primary text-primary-foreground'
: 'bg-secondary hover:bg-secondary/80'
}`}
style={
selectedEnseigne?.id === enseigne.id
? { backgroundColor: enseigne.couleur, color: 'white' }
: {}
}
>
{enseigne.nom}
{enseigne.catalogues_actifs_count > 0 && (
<span className="ml-2 px-2 py-0.5 rounded-full bg-white/20 text-xs">
{enseigne.catalogues_actifs_count}
</span>
)}
</button>
))}
</div>
</div>
{/* Catalogues Grid */}
{loading ? (
<div className="flex justify-center items-center py-20">
<Loader2 className="h-8 w-8 animate-spin text-primary" />
</div>
) : catalogues.length === 0 ? (
<div className="text-center py-20">
<BookOpen className="h-12 w-12 mx-auto mb-4 text-muted-foreground" />
<h3 className="text-lg font-medium mb-2">Aucun catalogue trouvé</h3>
<p className="text-muted-foreground">
{searchQuery
? 'Aucun catalogue ne correspond à votre recherche'
: 'Aucun catalogue disponible pour le moment'}
</p>
</div>
) : (
<>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6">
{catalogues.map(catalogue => (
<div
key={catalogue.id}
className="group bg-card rounded-lg border overflow-hidden hover:shadow-lg transition-all cursor-pointer"
onClick={() => openCatalogue(catalogue)}
>
{/* Cover Image */}
<div className="relative aspect-[3/4] overflow-hidden bg-secondary">
<img
src={catalogue.image_couverture_url}
alt={catalogue.titre}
className="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
/>
<div className="absolute inset-0 bg-gradient-to-t from-black/60 to-transparent opacity-0 group-hover:opacity-100 transition-opacity">
<div className="absolute bottom-4 left-4 right-4 text-white">
<Eye className="h-5 w-5 mx-auto mb-2" />
<p className="text-sm text-center">Voir le catalogue</p>
</div>
</div>
{isValidNow(catalogue) && (
<div className="absolute top-2 right-2">
<span className="px-2 py-1 rounded-full bg-green-500 text-white text-xs font-medium">
En cours
</span>
</div>
)}
</div>
{/* Content */}
<div className="p-4">
{/* Enseigne Badge */}
<div
className="inline-block px-3 py-1 rounded-full text-white text-xs font-medium mb-2"
style={{ backgroundColor: catalogue.enseigne.couleur }}
>
{catalogue.enseigne.nom}
</div>
{/* Title */}
<h3 className="font-semibold text-base mb-2 line-clamp-2">
{catalogue.titre}
</h3>
{/* Dates */}
<div className="flex items-center gap-1 text-sm text-muted-foreground mb-2">
<Calendar className="h-4 w-4" />
<span>
{formatDate(catalogue.date_debut)} - {formatDate(catalogue.date_fin)}
</span>
</div>
{/* Pages */}
<div className="flex items-center gap-1 text-sm text-muted-foreground">
<BookOpen className="h-4 w-4" />
<span>{catalogue.nombre_pages} pages</span>
</div>
</div>
</div>
))}
</div>
{/* Pagination */}
{pagination.total > pagination.limit && (
<div className="mt-8 flex justify-center items-center gap-2">
<button
onClick={() => setPagination(prev => ({ ...prev, page: prev.page - 1 }))}
disabled={pagination.page === 1}
className="px-4 py-2 rounded-lg border bg-background disabled:opacity-50"
>
Précédent
</button>
<span className="px-4 py-2 text-sm text-muted-foreground">
Page {pagination.page} sur {Math.ceil(pagination.total / pagination.limit)}
</span>
<button
onClick={() => setPagination(prev => ({ ...prev, page: prev.page + 1 }))}
disabled={pagination.page >= Math.ceil(pagination.total / pagination.limit)}
className="px-4 py-2 rounded-lg border bg-background disabled:opacity-50"
>
Suivant
</button>
</div>
)}
</>
)}
{/* Catalogue Viewer Modal */}
{selectedCatalogue && (
<div className="fixed inset-0 z-50 bg-black/80 flex items-center justify-center p-4">
<div className="bg-background rounded-lg max-w-4xl w-full max-h-[90vh] flex flex-col">
{/* Header */}
<div className="p-4 border-b flex justify-between items-start">
<div>
<h2 className="text-xl font-bold mb-1">{selectedCatalogue.titre}</h2>
<p className="text-sm text-muted-foreground">
{selectedCatalogue.enseigne.nom} • {cataloguePages.length} pages
</p>
</div>
<button
onClick={closeCatalogue}
className="px-4 py-2 rounded-lg hover:bg-secondary"
>
Fermer
</button>
</div>
{/* Pages Viewer */}
<div className="flex-1 overflow-y-auto p-4">
{cataloguePages.length === 0 ? (
<div className="flex justify-center py-20">
<Loader2 className="h-8 w-8 animate-spin text-primary" />
</div>
) : (
<div className="grid grid-cols-2 md:grid-cols-3 gap-4">
{cataloguePages.map(page => (
<div
key={page.id}
className="relative aspect-[3/4] rounded-lg overflow-hidden border cursor-pointer hover:shadow-lg transition-shadow"
onClick={() => window.open(page.image_url, '_blank')}
>
<img
src={page.image_url}
alt={`Page ${page.numero_page}`}
className="w-full h-full object-cover"
/>
<div className="absolute top-2 left-2 px-2 py-1 rounded bg-black/70 text-white text-xs font-medium">
Page {page.numero_page}
</div>
</div>
))}
</div>
)}
</div>
</div>
</div>
)}
</div>
);
}
+72
View File
@@ -0,0 +1,72 @@
# Setup and Migration Script for Bonial Module
#
# Run this script to set up the Bonial catalog module
Write-Host "================================================================" -ForegroundColor Cyan
Write-Host " BONIAL CATALOG MODULE - SETUP SCRIPT" -ForegroundColor Cyan
Write-Host "================================================================" -ForegroundColor Cyan
Write-Host ""
# Check if Docker is running
Write-Host "[1/3] Checking Docker..." -ForegroundColor Yellow
try {
$dockerCheck = docker ps 2>&1
if ($LASTEXITCODE -ne 0) {
Write-Host " ERROR: Docker is not running!" -ForegroundColor Red
Write-Host " Please start Docker Desktop and try again." -ForegroundColor Red
exit 1
}
Write-Host " OK - Docker is running" -ForegroundColor Green
} catch {
Write-Host " ERROR: Docker not found!" -ForegroundColor Red
Write-Host " Please install Docker Desktop." -ForegroundColor Red
exit 1
}
# Run Alembic migration inside container
Write-Host ""
Write-Host "[2/3] Running database migration..." -ForegroundColor Yellow
try {
docker compose exec -T priceflow alembic upgrade head
if ($LASTEXITCODE -eq 0) {
Write-Host " OK - Migration completed successfully" -ForegroundColor Green
} else {
Write-Host " WARNING: Migration may have failed" -ForegroundColor Yellow
Write-Host " Tables may already exist - this is normal" -ForegroundColor Yellow
}
} catch {
Write-Host " ERROR: Could not run migration" -ForegroundColor Red
Write-Host " Error: $_" -ForegroundColor Red
}
# Restart application to trigger seeding
Write-Host ""
Write-Host "[3/3] Restarting application to seed enseignes..." -ForegroundColor Yellow
try {
docker compose restart priceflow
if ($LASTEXITCODE -eq 0) {
Write-Host " OK - Application restarted" -ForegroundColor Green
Start-Sleep -Seconds 3
Write-Host " Check logs for: 'X enseigne(s) créée(s)'" -ForegroundColor Cyan
} else {
Write-Host " ERROR: Could not restart application" -ForegroundColor Red
}
} catch {
Write-Host " ERROR: $_" -ForegroundColor Red
}
Write-Host ""
Write-Host "================================================================" -ForegroundColor Cyan
Write-Host " SETUP COMPLETE!" -ForegroundColor Green
Write-Host "================================================================" -ForegroundColor Cyan
Write-Host ""
Write-Host "Next steps:" -ForegroundColor Yellow
Write-Host " 1. Check application logs:" -ForegroundColor White
Write-Host " docker compose logs -f priceflow" -ForegroundColor Gray
Write-Host ""
Write-Host " 2. Run verification script:" -ForegroundColor White
Write-Host " docker compose exec priceflow python verify_bonial.py" -ForegroundColor Gray
Write-Host ""
Write-Host " 3. Test API endpoints:" -ForegroundColor White
Write-Host " Invoke-WebRequest http://localhost:8555/api/catalogues/enseignes" -ForegroundColor Gray
Write-Host ""
+294
View File
@@ -0,0 +1,294 @@
"""
Verification Script for Bonial Catalog Module
Tests database setup, scraper functionality, and API endpoints.
"""
import asyncio
import logging
import sys
from datetime import datetime
import httpx
from sqlalchemy import inspect, text
from app.database import SessionLocal, engine
from app.models import Catalogue, CataloguePage, Enseigne, ScrapingLog
from app.services.bonial_scraper import scrape_enseigne
from app.services.seed_enseignes import seed_enseignes
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
API_BASE_URL = "http://localhost:8555/api"
def check_database_tables():
"""Verify that catalog module tables exist."""
logger.info("=" * 60)
logger.info("Phase 1: Checking Database Tables")
logger.info("=" * 60)
inspector = inspect(engine)
required_tables = [
"enseignes",
"catalogues",
"catalogue_pages",
"scraping_logs",
]
existing_tables = inspector.get_table_names()
all_exist = True
for table in required_tables:
exists = table in existing_tables
status = "✅" if exists else "❌"
logger.info(f"{status} Table '{table}': {'EXISTS' if exists else 'MISSING'}")
if not exists:
all_exist = False
if all_exist:
logger.info("\n✅ All required tables exist!\n")
return True
else:
logger.error("\n❌ Some tables are missing. Run migration: alembic upgrade head\n")
return False
def check_enseignes_seeding():
"""Verify that enseignes are seeded."""
logger.info("=" * 60)
logger.info("Phase 2: Checking Enseignes Seeding")
logger.info("=" * 60)
db = SessionLocal()
try:
count = db.query(Enseigne).count()
logger.info(f"Found {count} enseignes in database")
if count == 0:
logger.info("Seeding enseignes...")
created = seed_enseignes(db)
logger.info(f"✅ Created {created} enseignes")
count = created
if count >= 9:
logger.info("\n✅ All 9 enseignes are seeded!\n")
# Display enseignes
enseignes = db.query(Enseigne).order_by(Enseigne.ordre_affichage).all()
for ens in enseignes:
active = "✅" if ens.is_active else "⚠️"
logger.info(f" {active} {ens.ordre_affichage}. {ens.nom} (slug: {ens.slug_bonial})")
return True
else:
logger.warning(f"\n⚠️ Expected 9 enseignes, found {count}\n")
return False
finally:
db.close()
async def test_scraper():
"""Test the Bonial scraper on one enseigne."""
logger.info("=" * 60)
logger.info("Phase 3: Testing Bonial Scraper (Gifi)")
logger.info("=" * 60)
db = SessionLocal()
try:
# Get Gifi enseigne
gifi = db.query(Enseigne).filter_by(slug_bonial="Gifi").first()
if not gifi:
logger.error("❌ Gifi enseigne not found")
return False
logger.info(f"Testing scraper for: {gifi.nom}")
logger.info("This may take 30-60 seconds...")
# Run scraper
log = await scrape_enseigne(gifi, db)
# Display results
logger.info(f"\nScraping completed:")
logger.info(f" Status: {log.statut}")
logger.info(f" Catalogues found: {log.catalogues_trouves}")
logger.info(f" New catalogues: {log.catalogues_nouveaux}")
logger.info(f" Duration: {log.duree_secondes:.2f}s")
if log.message_erreur:
logger.warning(f" Error: {log.message_erreur}")
if log.statut in ["success", "partial"] and log.catalogues_trouves > 0:
logger.info("\n✅ Scraper is working!\n")
# Display sample catalog
cat = db.query(Catalogue).filter_by(enseigne_id=gifi.id).first()
if cat:
logger.info(f"Sample catalog:")
logger.info(f" Titre: {cat.titre}")
logger.info(f" Dates: {cat.date_debut.date()} → {cat.date_fin.date()}")
logger.info(f" Pages: {cat.nombre_pages}")
return True
else:
logger.error("\n❌ Scraper failed or found no catalogs\n")
return False
except Exception as e:
logger.error(f"❌ Error testing scraper: {e}")
return False
finally:
db.close()
async def test_api_endpoints():
"""Test API endpoints."""
logger.info("=" * 60)
logger.info("Phase 4: Testing API Endpoints")
logger.info("=" * 60)
async with httpx.AsyncClient(timeout=30.0) as client:
# Test 1: Get enseignes
logger.info("\n1. Testing GET /api/catalogues/enseignes")
try:
response = await client.get(f"{API_BASE_URL}/catalogues/enseignes")
if response.status_code == 200:
enseignes = response.json()
logger.info(f" ✅ Status 200 - Found {len(enseignes)} enseignes")
if enseignes:
logger.info(f" Sample: {enseignes[0]['nom']} ({enseignes[0]['catalogues_actifs_count']} catalogues)")
else:
logger.error(f" ❌ Status {response.status_code}")
return False
except Exception as e:
logger.error(f" ❌ Error: {e}")
return False
# Test 2: Get catalogues
logger.info("\n2. Testing GET /api/catalogues")
try:
response = await client.get(f"{API_BASE_URL}/catalogues?page=1&limit=5")
if response.status_code == 200:
data = response.json()
catalogues = data.get("data", [])
pagination = data.get("pagination", {})
logger.info(f" ✅ Status 200 - Found {pagination.get('total', 0)} catalogues")
logger.info(f" Page: {pagination.get('page')}/{pagination.get('pages_total')}")
if catalogues:
logger.info(f" Sample: {catalogues[0]['titre']}")
else:
logger.error(f" ❌ Status {response.status_code}")
return False
except Exception as e:
logger.error(f" ❌ Error: {e}")
return False
# Test 3: Get catalogue detail
logger.info("\n3. Testing GET /api/catalogues/{id}")
try:
# Get first catalog ID
db = SessionLocal()
cat = db.query(Catalogue).first()
db.close()
if cat:
response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}")
if response.status_code == 200:
detail = response.json()
logger.info(f" ✅ Status 200 - Catalogue: {detail['titre']}")
logger.info(f" Pages: {detail['nombre_pages']}")
else:
logger.error(f" ❌ Status {response.status_code}")
return False
else:
logger.warning(" ⚠️ No catalogues in DB to test detail endpoint")
except Exception as e:
logger.error(f" ❌ Error: {e}")
return False
# Test 4: Get catalogue pages
logger.info("\n4. Testing GET /api/catalogues/{id}/pages")
try:
if cat:
response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}/pages")
if response.status_code == 200:
pages = response.json()
logger.info(f" ✅ Status 200 - Found {len(pages)} pages")
if pages:
logger.info(f" Sample page: {pages[0]['numero_page']} - {pages[0]['image_url'][:50]}...")
else:
logger.error(f" ❌ Status {response.status_code}")
return False
except Exception as e:
logger.error(f" ❌ Error: {e}")
return False
# Test 5: Get stats (requires working DB)
logger.info("\n5. Testing GET /api/catalogues/admin/stats")
try:
response = await client.get(f"{API_BASE_URL}/catalogues/admin/stats")
if response.status_code == 200:
stats = response.json()
logger.info(f" ✅ Status 200 - Total catalogues: {stats['total_catalogues']}")
logger.info(f" Prochaine exécution: {stats['prochaine_execution']}")
else:
logger.error(f" ❌ Status {response.status_code}")
# Stats is optional, don't fail
except Exception as e:
logger.warning(f" ⚠️ Stats endpoint error (may require auth): {e}")
logger.info("\n✅ All API endpoints are working!\n")
return True
async def main():
"""Run all verification tests."""
logger.info("\n" + "=" * 60)
logger.info("BONIAL CATALOG MODULE - VERIFICATION SCRIPT")
logger.info("=" * 60 + "\n")
results = []
# Phase 1: Database tables
results.append(("Database Tables", check_database_tables()))
if not results[0][1]:
logger.error("\n❌ Database not ready. Please run: alembic upgrade head")
sys.exit(1)
# Phase 2: Enseignes seeding
results.append(("Enseignes Seeding", check_enseignes_seeding()))
# Phase 3: Scraper test (optional, can be slow)
scraper_test = input("\nRun scraper test? (Gifi - takes ~60s) [y/N]: ").lower() == "y"
if scraper_test:
results.append(("Scraper Test", await test_scraper()))
# Phase 4: API endpoints (requires app to be running)
api_test = input("\nTest API endpoints? (App must be running on :8555) [y/N]: ").lower() == "y"
if api_test:
results.append(("API Endpoints", await test_api_endpoints()))
# Summary
logger.info("\n" + "=" * 60)
logger.info("VERIFICATION SUMMARY")
logger.info("=" * 60)
for name, passed in results:
status = "✅ PASSED" if passed else "❌ FAILED"
logger.info(f"{status}: {name}")
all_passed = all(result[1] for result in results)
if all_passed:
logger.info("\n✅ ALL TESTS PASSED - Bonial module is ready!")
else:
logger.error("\n❌ SOME TESTS FAILED - Please review errors above")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(main())