mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: Add initial application structure, core data models, and a new catalog module with Bonial scraping capabilities.
This commit is contained in:
1 parent
906c6b483d
commit
58e6be3f0d
17 files changed
+2483
-7
No files matched your search
@@ -29,8 +29,9 @@
|
||||
- **Historique visuel** : Conserve l'historique des captures d'écran avec les métadonnées IA
|
||||
- **Défilement intelligent** : Scroll automatique pour charger le contenu différé
|
||||
- **Notifications** : Support multi-canaux (Discord, Telegram, Email, etc.) via [Apprise](https://github.com/caronc/apprise)
|
||||
- **Catalogues promotionnels** : Consultation des catalogues et prospectus discount (Gifi, Action, Centrakor, etc.) scrapés depuis Bonial.fr
|
||||
- **Interface en français** : Application entièrement traduite en français
|
||||
- **Mode sombre** : Interface moderne avec support du thème clair/sombre
|
||||
- **Mode sombre** : Interface moderne avec support du thème clair/foncé
|
||||
- **Dockerisé** : Déploiement facile avec Docker Compose
|
||||
|
||||
## Prérequis
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Add catalog module tables
|
||||
|
||||
Revision ID: c1d2e3f4g5h6
|
||||
Revises: b2c3d4e5f6g7
|
||||
Create Date: 2025-11-29 18:56:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c1d2e3f4g5h6"
|
||||
down_revision: str | None = "b2c3d4e5f6g7"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create tables for the catalog module."""
|
||||
|
||||
# Create enseignes table
|
||||
op.create_table(
|
||||
"enseignes",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, index=True),
|
||||
sa.Column("nom", sa.String(), nullable=False),
|
||||
sa.Column("slug_bonial", sa.String(), nullable=False, unique=True, index=True),
|
||||
sa.Column("logo_url", sa.String(), nullable=True),
|
||||
sa.Column("couleur", sa.String(), nullable=False),
|
||||
sa.Column("site_url", sa.String(), nullable=True),
|
||||
sa.Column("description", sa.String(), nullable=True),
|
||||
sa.Column("is_active", sa.Boolean(), default=True),
|
||||
sa.Column("ordre_affichage", sa.Integer(), default=0),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=True),
|
||||
)
|
||||
|
||||
# Create catalogues table
|
||||
op.create_table(
|
||||
"catalogues",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, index=True),
|
||||
sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=False, index=True),
|
||||
sa.Column("titre", sa.String(), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("date_debut", sa.DateTime(), nullable=False, index=True),
|
||||
sa.Column("date_fin", sa.DateTime(), nullable=False, index=True),
|
||||
sa.Column("image_couverture_url", sa.String(), nullable=False),
|
||||
sa.Column("catalogue_url", sa.String(), nullable=False),
|
||||
sa.Column("nombre_pages", sa.Integer(), default=0),
|
||||
sa.Column("statut", sa.String(), default="actif", index=True),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("metadonnees", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=True),
|
||||
)
|
||||
|
||||
# Create catalogue_pages table
|
||||
op.create_table(
|
||||
"catalogue_pages",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, index=True),
|
||||
sa.Column("catalogue_id", sa.Integer(), sa.ForeignKey("catalogues.id"), nullable=False, index=True),
|
||||
sa.Column("numero_page", sa.Integer(), nullable=False),
|
||||
sa.Column("image_url", sa.String(), nullable=False),
|
||||
sa.Column("image_thumbnail_url", sa.String(), nullable=True),
|
||||
sa.Column("largeur", sa.Integer(), nullable=True),
|
||||
sa.Column("hauteur", sa.Integer(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=True),
|
||||
)
|
||||
|
||||
# Create unique index on catalogue_id + numero_page
|
||||
op.create_index(
|
||||
"idx_catalogue_page_numero",
|
||||
"catalogue_pages",
|
||||
["catalogue_id", "numero_page"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
# Create scraping_logs table
|
||||
op.create_table(
|
||||
"scraping_logs",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, index=True),
|
||||
sa.Column("date_execution", sa.DateTime(), nullable=True, index=True),
|
||||
sa.Column("enseigne_id", sa.Integer(), sa.ForeignKey("enseignes.id"), nullable=True),
|
||||
sa.Column("statut", sa.String(), nullable=False),
|
||||
sa.Column("catalogues_trouves", sa.Integer(), default=0),
|
||||
sa.Column("catalogues_nouveaux", sa.Integer(), default=0),
|
||||
sa.Column("catalogues_mis_a_jour", sa.Integer(), default=0),
|
||||
sa.Column("duree_secondes", sa.Float(), nullable=True),
|
||||
sa.Column("message_erreur", sa.Text(), nullable=True),
|
||||
sa.Column("details", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop catalog module tables."""
|
||||
op.drop_table("scraping_logs")
|
||||
op.drop_index("idx_catalogue_page_numero", table_name="catalogue_pages")
|
||||
op.drop_table("catalogue_pages")
|
||||
op.drop_table("catalogues")
|
||||
op.drop_table("enseignes")
|
||||
+20
-3
@@ -13,9 +13,10 @@ from sqlalchemy import text
|
||||
|
||||
from app.database import SessionLocal, engine
|
||||
from app.limiter import limiter
|
||||
from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug
|
||||
from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues
|
||||
from app.services.scheduler_service import scheduled_refresh, scheduler
|
||||
from app.services import auth_service, search_service
|
||||
from app.services import auth_service, search_service, seed_enseignes
|
||||
from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
@@ -119,6 +120,13 @@ async def lifespan(app: FastAPI):
|
||||
logger.info("Utilisateur admin par défaut créé (admin/admin)")
|
||||
else:
|
||||
logger.info("Utilisateurs déjà initialisés")
|
||||
|
||||
# Seed enseignes for catalog module
|
||||
created_enseignes = seed_enseignes.seed_enseignes(db)
|
||||
if created_enseignes > 0:
|
||||
logger.info(f"{created_enseignes} enseigne(s) de catalogues créée(s)")
|
||||
else:
|
||||
logger.info("Enseignes déjà initialisés")
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
@@ -127,10 +135,16 @@ async def lifespan(app: FastAPI):
|
||||
logger.info("Starting smart scheduler (Heartbeat: 1 minute)")
|
||||
scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True)
|
||||
scheduler.start()
|
||||
|
||||
# Start catalog scraping scheduler
|
||||
logger.info("Starting catalog scraping scheduler (6h and 18h daily)")
|
||||
start_catalog_scheduler()
|
||||
|
||||
logger.info("Application started")
|
||||
yield
|
||||
logger.info("Shutting down scheduler...")
|
||||
logger.info("Shutting down schedulers...")
|
||||
scheduler.shutdown(wait=True)
|
||||
stop_catalog_scheduler()
|
||||
logger.info("Application shutdown complete")
|
||||
|
||||
|
||||
@@ -170,6 +184,9 @@ for router in [notifications.router, items.router, settings.router, jobs.router,
|
||||
app.include_router(search.router)
|
||||
app.include_router(search_sites.router)
|
||||
|
||||
# Catalog router (already has /api prefix)
|
||||
app.include_router(catalogues.router)
|
||||
|
||||
# Auth router (already has /api prefix)
|
||||
app.include_router(auth.router)
|
||||
|
||||
|
||||
@@ -113,3 +113,87 @@ class NotificationChannel(Base):
|
||||
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
|
||||
|
||||
items = relationship("Item", back_populates="notification_channel")
|
||||
|
||||
|
||||
# === CATALOG MODULE MODELS ===
|
||||
|
||||
class Enseigne(Base):
|
||||
"""Stores/brands for promotional catalogs"""
|
||||
__tablename__ = "enseignes"
|
||||
|
||||
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
|
||||
nom: str = Column(String, nullable=False) # type: ignore # "Gifi"
|
||||
slug_bonial: str = Column(String, unique=True, nullable=False, index=True) # type: ignore # "Gifi"
|
||||
logo_url: str | None = Column(String, nullable=True) # type: ignore
|
||||
couleur: str = Column(String, nullable=False) # type: ignore # "#E30613"
|
||||
site_url: str | None = Column(String, nullable=True) # type: ignore
|
||||
description: str | None = Column(String, nullable=True) # type: ignore
|
||||
is_active: bool = Column(Boolean, default=True) # type: ignore
|
||||
ordre_affichage: int = Column(Integer, default=0) # type: ignore
|
||||
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
|
||||
updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore
|
||||
|
||||
# Relationships
|
||||
catalogues = relationship("Catalogue", back_populates="enseigne", cascade="all, delete-orphan")
|
||||
scraping_logs = relationship("ScrapingLog", back_populates="enseigne")
|
||||
|
||||
|
||||
class Catalogue(Base):
|
||||
"""Promotional catalogs from stores"""
|
||||
__tablename__ = "catalogues"
|
||||
|
||||
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
|
||||
enseigne_id: int = Column(Integer, ForeignKey("enseignes.id"), nullable=False, index=True) # type: ignore
|
||||
titre: str = Column(String, nullable=False) # type: ignore
|
||||
description: str | None = Column(Text, nullable=True) # type: ignore
|
||||
date_debut: datetime = Column(DateTime, nullable=False, index=True) # type: ignore
|
||||
date_fin: datetime = Column(DateTime, nullable=False, index=True) # type: ignore
|
||||
image_couverture_url: str = Column(String, nullable=False) # type: ignore
|
||||
catalogue_url: str = Column(String, nullable=False) # type: ignore # URL Bonial viewer
|
||||
nombre_pages: int = Column(Integer, default=0) # type: ignore
|
||||
statut: str = Column(String, default="actif", index=True) # type: ignore # actif, termine, erreur
|
||||
content_hash: str = Column(String(64), unique=True, nullable=False, index=True) # type: ignore # SHA256
|
||||
metadonnees: str | None = Column(Text, nullable=True) # type: ignore # JSON for additional data
|
||||
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
|
||||
updated_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # type: ignore
|
||||
|
||||
# Relationships
|
||||
enseigne = relationship("Enseigne", back_populates="catalogues")
|
||||
pages = relationship("CataloguePage", back_populates="catalogue", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class CataloguePage(Base):
|
||||
"""Individual pages of a catalog"""
|
||||
__tablename__ = "catalogue_pages"
|
||||
|
||||
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
|
||||
catalogue_id: int = Column(Integer, ForeignKey("catalogues.id"), nullable=False, index=True) # type: ignore
|
||||
numero_page: int = Column(Integer, nullable=False) # type: ignore
|
||||
image_url: str = Column(String, nullable=False) # type: ignore
|
||||
image_thumbnail_url: str | None = Column(String, nullable=True) # type: ignore
|
||||
largeur: int | None = Column(Integer, nullable=True) # type: ignore
|
||||
hauteur: int | None = Column(Integer, nullable=True) # type: ignore
|
||||
created_at: datetime = Column(DateTime, default=lambda: datetime.now(UTC)) # type: ignore
|
||||
|
||||
# Relationship
|
||||
catalogue = relationship("Catalogue", back_populates="pages")
|
||||
|
||||
|
||||
class ScrapingLog(Base):
|
||||
"""Logs of scraping executions"""
|
||||
__tablename__ = "scraping_logs"
|
||||
|
||||
id: int = Column(Integer, primary_key=True, index=True) # type: ignore
|
||||
date_execution: datetime = Column(DateTime, default=lambda: datetime.now(UTC), index=True) # type: ignore
|
||||
enseigne_id: int | None = Column(Integer, ForeignKey("enseignes.id"), nullable=True) # type: ignore # NULL = all stores
|
||||
statut: str = Column(String, nullable=False) # type: ignore # success, error, partial
|
||||
catalogues_trouves: int = Column(Integer, default=0) # type: ignore
|
||||
catalogues_nouveaux: int = Column(Integer, default=0) # type: ignore
|
||||
catalogues_mis_a_jour: int = Column(Integer, default=0) # type: ignore
|
||||
duree_secondes: float | None = Column(Float, nullable=True) # type: ignore
|
||||
message_erreur: str | None = Column(Text, nullable=True) # type: ignore
|
||||
details: str | None = Column(Text, nullable=True) # type: ignore # JSON
|
||||
|
||||
# Relationship
|
||||
enseigne = relationship("Enseigne", back_populates="scraping_logs")
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""
|
||||
API Router for catalog module
|
||||
Endpoints for enseignes, catalogues, and admin operations
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, and_
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
|
||||
from app.schemas_catalogues import (
|
||||
EnseigneResponse,
|
||||
CatalogueListResponse,
|
||||
CatalogueDetailResponse,
|
||||
CataloguePageResponse,
|
||||
CataloguesPaginatedResponse,
|
||||
PaginationMeta,
|
||||
ScrapingLogResponse,
|
||||
ScrapingStatsResponse,
|
||||
)
|
||||
from app.services.bonial_scraper import scrape_all_enseignes, scrape_enseigne
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/catalogues", tags=["catalogues"])
|
||||
|
||||
|
||||
# === Enseignes Endpoints ===
|
||||
|
||||
@router.get("/enseignes", response_model=list[EnseigneResponse])
|
||||
async def get_enseignes(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get all active enseignes with catalog counts"""
|
||||
enseignes = (
|
||||
db.query(Enseigne)
|
||||
.filter(Enseigne.is_active == True)
|
||||
.order_by(Enseigne.ordre_affichage)
|
||||
.all()
|
||||
)
|
||||
|
||||
# Add catalog counts
|
||||
result = []
|
||||
for enseigne in enseignes:
|
||||
count = (
|
||||
db.query(func.count(Catalogue.id))
|
||||
.filter(
|
||||
and_(
|
||||
Catalogue.enseigne_id == enseigne.id,
|
||||
Catalogue.statut == "actif"
|
||||
)
|
||||
)
|
||||
.scalar()
|
||||
)
|
||||
|
||||
enseigne_data = EnseigneResponse.model_validate(enseigne)
|
||||
enseigne_data.catalogues_actifs_count = count
|
||||
result.append(enseigne_data)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# === Catalogues Endpoints ===
|
||||
|
||||
@router.get("", response_model=CataloguesPaginatedResponse)
|
||||
async def get_catalogues(
|
||||
enseigne_ids: str | None = Query(None, description="Comma-separated enseigne IDs"),
|
||||
statut: str | None = Query(None, description="Filter by status: actif, termine, erreur, tous"),
|
||||
date_debut_min: datetime | None = Query(None, description="Start date minimum"),
|
||||
date_fin_max: datetime | None = Query(None, description="End date maximum"),
|
||||
recherche: str | None = Query(None, description="Search in titles"),
|
||||
page: int = Query(1, ge=1, description="Page number"),
|
||||
limit: int = Query(20, ge=1, le=100, description="Items per page"),
|
||||
sort: str = Query("date_debut", description="Sort by: date_debut, date_fin, enseigne"),
|
||||
order: str = Query("desc", description="Order: asc, desc"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get catalogues with filters and pagination"""
|
||||
|
||||
# Build query
|
||||
query = db.query(Catalogue)
|
||||
|
||||
# Filter by enseignes
|
||||
if enseigne_ids:
|
||||
try:
|
||||
ids = [int(id_str.strip()) for id_str in enseigne_ids.split(",")]
|
||||
query = query.filter(Catalogue.enseigne_id.in_(ids))
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid enseigne_ids format")
|
||||
|
||||
# Filter by status
|
||||
if statut and statut != "tous":
|
||||
query = query.filter(Catalogue.statut == statut)
|
||||
|
||||
# Filter by dates
|
||||
if date_debut_min:
|
||||
query = query.filter(Catalogue.date_debut >= date_debut_min)
|
||||
if date_fin_max:
|
||||
query = query.filter(Catalogue.date_fin <= date_fin_max)
|
||||
|
||||
# Search in titles
|
||||
if recherche:
|
||||
query = query.filter(Catalogue.titre.ilike(f"%{recherche}%"))
|
||||
|
||||
# Sort
|
||||
sort_column = {
|
||||
"date_debut": Catalogue.date_debut,
|
||||
"date_fin": Catalogue.date_fin,
|
||||
"enseigne": Catalogue.enseigne_id,
|
||||
}.get(sort, Catalogue.date_debut)
|
||||
|
||||
if order == "desc":
|
||||
query = query.order_by(sort_column.desc())
|
||||
else:
|
||||
query = query.order_by(sort_column.asc())
|
||||
|
||||
# Count total
|
||||
total = query.count()
|
||||
|
||||
# Paginate
|
||||
offset = (page - 1) * limit
|
||||
catalogues = query.offset(offset).limit(limit).all()
|
||||
|
||||
# Build response
|
||||
data = []
|
||||
for catalogue in catalogues:
|
||||
cat_response = CatalogueListResponse.model_validate(catalogue)
|
||||
data.append(cat_response)
|
||||
|
||||
pagination = PaginationMeta(
|
||||
total=total,
|
||||
page=page,
|
||||
limit=limit,
|
||||
pages_total=(total + limit - 1) // limit,
|
||||
)
|
||||
|
||||
# Get last scraping time
|
||||
last_scraping = (
|
||||
db.query(ScrapingLog)
|
||||
.order_by(ScrapingLog.date_execution.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
metadata = {
|
||||
"derniere_mise_a_jour": last_scraping.date_execution if last_scraping else None
|
||||
}
|
||||
|
||||
return CataloguesPaginatedResponse(
|
||||
data=data,
|
||||
pagination=pagination,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{catalogue_id}", response_model=CatalogueDetailResponse)
|
||||
async def get_catalogue_detail(
|
||||
catalogue_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get detailed information for a specific catalogue"""
|
||||
catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
|
||||
|
||||
if not catalogue:
|
||||
raise HTTPException(status_code=404, detail="Catalogue not found")
|
||||
|
||||
return CatalogueDetailResponse.model_validate(catalogue)
|
||||
|
||||
|
||||
@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse])
|
||||
async def get_catalogue_pages(
|
||||
catalogue_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get all pages of a catalogue"""
|
||||
# Verify catalogue exists
|
||||
catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
|
||||
|
||||
if not catalogue:
|
||||
raise HTTPException(status_code=404, detail="Catalogue not found")
|
||||
|
||||
# Get pages ordered by page number
|
||||
pages = (
|
||||
db.query(CataloguePage)
|
||||
.filter(CataloguePage.catalogue_id == catalogue_id)
|
||||
.order_by(CataloguePage.numero_page)
|
||||
.all()
|
||||
)
|
||||
|
||||
return [CataloguePageResponse.model_validate(page) for page in pages]
|
||||
|
||||
|
||||
# === Admin Endpoints ===
|
||||
|
||||
@router.post("/admin/scraping/trigger", response_model=dict)
|
||||
async def trigger_scraping(
|
||||
enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Manually trigger catalog scraping"""
|
||||
|
||||
if enseigne_id:
|
||||
# Scrape single enseigne
|
||||
enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first()
|
||||
|
||||
if not enseigne:
|
||||
raise HTTPException(status_code=404, detail="Enseigne not found")
|
||||
|
||||
# Run scraping asynchronously (in background)
|
||||
import asyncio
|
||||
from app.services.bonial_scraper import scrape_enseigne as scrape_ens
|
||||
|
||||
log = await scrape_ens(enseigne, db)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Scraping complete for {enseigne.nom}",
|
||||
"catalogues_trouves": log.catalogues_trouves,
|
||||
"catalogues_nouveaux": log.catalogues_nouveaux,
|
||||
}
|
||||
else:
|
||||
# Scrape all enseignes
|
||||
logs = await scrape_all_enseignes(db)
|
||||
|
||||
total_new = sum(log.catalogues_nouveaux for log in logs)
|
||||
total_found = sum(log.catalogues_trouves for log in logs)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Scraping complete for all enseignes",
|
||||
"enseignes_processed": len(logs),
|
||||
"catalogues_trouves": total_found,
|
||||
"catalogues_nouveaux": total_new,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse])
|
||||
async def get_scraping_logs(
|
||||
enseigne_id: int | None = Query(None, description="Filter by enseigne"),
|
||||
statut: str | None = Query(None, description="Filter by status"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Number of logs to return"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get scraping execution logs"""
|
||||
query = db.query(ScrapingLog)
|
||||
|
||||
if enseigne_id:
|
||||
query = query.filter(ScrapingLog.enseigne_id == enseigne_id)
|
||||
|
||||
if statut:
|
||||
query = query.filter(ScrapingLog.statut == statut)
|
||||
|
||||
logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all()
|
||||
|
||||
# Add enseigne name
|
||||
result = []
|
||||
for log in logs:
|
||||
log_dict = ScrapingLogResponse.model_validate(log).model_dump()
|
||||
if log.enseigne_id:
|
||||
enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first()
|
||||
log_dict["enseigne_nom"] = enseigne.nom if enseigne else None
|
||||
else:
|
||||
log_dict["enseigne_nom"] = "Toutes"
|
||||
|
||||
result.append(ScrapingLogResponse(**log_dict))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/admin/stats", response_model=ScrapingStatsResponse)
|
||||
async def get_scraping_stats(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get scraping statistics"""
|
||||
|
||||
# Total catalogues
|
||||
total_catalogues = db.query(func.count(Catalogue.id)).scalar()
|
||||
|
||||
# Catalogues par enseigne
|
||||
catalogues_par_enseigne = {}
|
||||
enseignes = db.query(Enseigne).all()
|
||||
|
||||
for enseigne in enseignes:
|
||||
count = (
|
||||
db.query(func.count(Catalogue.id))
|
||||
.filter(Catalogue.enseigne_id == enseigne.id)
|
||||
.scalar()
|
||||
)
|
||||
catalogues_par_enseigne[enseigne.nom] = count
|
||||
|
||||
# Last scraping time
|
||||
last_log = (
|
||||
db.query(ScrapingLog)
|
||||
.order_by(ScrapingLog.date_execution.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
derniere_mise_a_jour = last_log.date_execution if last_log else None
|
||||
|
||||
# Next execution (6:00 or 18:00)
|
||||
now = datetime.now()
|
||||
if now.hour < 6:
|
||||
prochaine = f"Aujourd'hui à 6:00"
|
||||
elif now.hour < 18:
|
||||
prochaine = f"Aujourd'hui à 18:00"
|
||||
else:
|
||||
prochaine = f"Demain à 6:00"
|
||||
|
||||
return ScrapingStatsResponse(
|
||||
total_catalogues=total_catalogues,
|
||||
catalogues_par_enseigne=catalogues_par_enseigne,
|
||||
derniere_mise_a_jour=derniere_mise_a_jour,
|
||||
prochaine_execution=prochaine,
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Pydantic schemas for catalog module API
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
# === Enseigne Schemas ===
|
||||
|
||||
class EnseigneResponse(BaseModel):
|
||||
id: int
|
||||
nom: str
|
||||
slug_bonial: str
|
||||
logo_url: str | None
|
||||
couleur: str
|
||||
site_url: str | None
|
||||
description: str | None
|
||||
is_active: bool
|
||||
ordre_affichage: int
|
||||
|
||||
# Statistics
|
||||
catalogues_actifs_count: int | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# === Catalogue Schemas ===
|
||||
|
||||
class CatalogueListResponse(BaseModel):
|
||||
id: int
|
||||
enseigne: EnseigneResponse
|
||||
titre: str
|
||||
date_debut: datetime
|
||||
date_fin: datetime
|
||||
image_couverture_url: str
|
||||
statut: str
|
||||
nombre_pages: int
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class CatalogueDetailResponse(BaseModel):
|
||||
id: int
|
||||
enseigne: EnseigneResponse
|
||||
titre: str
|
||||
description: str | None
|
||||
date_debut: datetime
|
||||
date_fin: datetime
|
||||
image_couverture_url: str
|
||||
catalogue_url: str
|
||||
statut: str
|
||||
nombre_pages: int
|
||||
metadonnees: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class CataloguePageResponse(BaseModel):
|
||||
id: int
|
||||
numero_page: int
|
||||
image_url: str
|
||||
image_thumbnail_url: str | None
|
||||
largeur: int | None
|
||||
hauteur: int | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# === Pagination ===
|
||||
|
||||
class PaginationMeta(BaseModel):
|
||||
total: int
|
||||
page: int
|
||||
limit: int
|
||||
pages_total: int
|
||||
|
||||
|
||||
class CataloguesPaginatedResponse(BaseModel):
|
||||
data: list[CatalogueListResponse]
|
||||
pagination: PaginationMeta
|
||||
metadata: dict
|
||||
|
||||
|
||||
# === Admin Schemas ===
|
||||
|
||||
class ScrapingLogResponse(BaseModel):
|
||||
id: int
|
||||
date_execution: datetime
|
||||
enseigne_id: int | None
|
||||
enseigne_nom: str | None
|
||||
statut: str
|
||||
catalogues_trouves: int
|
||||
catalogues_nouveaux: int
|
||||
catalogues_mis_a_jour: int
|
||||
duree_secondes: float | None
|
||||
message_erreur: str | None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ScrapingStatsResponse(BaseModel):
|
||||
total_catalogues: int
|
||||
catalogues_par_enseigne: dict[str, int]
|
||||
derniere_mise_a_jour: datetime | None
|
||||
prochaine_execution: str | None
|
||||
@@ -0,0 +1,400 @@
|
||||
"""
|
||||
Bonial.fr Scraper Service
|
||||
|
||||
Scrapes promotional catalogs from Bonial.fr for configured enseignes.
|
||||
Uses Playwright for browser automation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from playwright.async_api import async_playwright, Page, Browser
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Bonial selectors based on manual analysis
|
||||
BONIAL_SELECTORS = {
|
||||
"catalog_card": "section:has(img[src*='content-media.bonial.biz'])",
|
||||
"catalog_title": "div.text-dark.text-lg.font-bold, h3, h2",
|
||||
"catalog_image": "img[src*='content-media.bonial.biz']",
|
||||
"catalog_link": "a:has-text('Ouvrir le catalogue')",
|
||||
"cookie_accept": "button:has-text('OK'), button:has-text('Accepter')",
|
||||
}
|
||||
|
||||
# Date pattern: "mar. 25/11 - lun. 08/12/2025"
|
||||
DATE_PATTERN = r"(\d{2}/\d{2})\s*-\s*(\d{2}/\d{2}/\d{4})"
|
||||
|
||||
|
||||
def parse_bonial_dates(date_str: str) -> tuple[datetime, datetime]:
|
||||
"""
|
||||
Parse Bonial date format to datetime objects.
|
||||
|
||||
Args:
|
||||
date_str: Date string like "mar. 25/11 - lun. 08/12/2025"
|
||||
|
||||
Returns:
|
||||
Tuple of (date_debut, date_fin)
|
||||
|
||||
Raises:
|
||||
ValueError: If date format is invalid
|
||||
"""
|
||||
match = re.search(DATE_PATTERN, date_str)
|
||||
|
||||
if not match:
|
||||
raise ValueError(f"Invalid date format: {date_str}")
|
||||
|
||||
debut_str, fin_str = match.groups()
|
||||
|
||||
# Extract year from end date
|
||||
year = fin_str.split("/")[2]
|
||||
|
||||
# Add year to start date
|
||||
debut_full = f"{debut_str}/{year}"
|
||||
|
||||
# Parse dates
|
||||
debut = datetime.strptime(debut_full, "%d/%m/%Y")
|
||||
fin = datetime.strptime(fin_str, "%d/%m/%Y")
|
||||
|
||||
return debut, fin
|
||||
|
||||
|
||||
def compute_catalog_hash(enseigne_id: int, titre: str, date_debut: datetime) -> str:
|
||||
"""
|
||||
Compute SHA256 hash for duplicate detection.
|
||||
|
||||
Args:
|
||||
enseigne_id: ID of the enseigne
|
||||
titre: Catalog title
|
||||
date_debut: Start date
|
||||
|
||||
Returns:
|
||||
SHA256 hash string
|
||||
"""
|
||||
content = f"{enseigne_id}|{titre}|{date_debut.isoformat()}"
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
|
||||
async def accept_cookies(page: Page) -> None:
|
||||
"""Accept cookie consent banner if present."""
|
||||
try:
|
||||
await page.click(BONIAL_SELECTORS["cookie_accept"], timeout=3000)
|
||||
logger.debug("Accepted cookies")
|
||||
await asyncio.sleep(1)
|
||||
except Exception:
|
||||
logger.debug("No cookie banner found or already accepted")
|
||||
|
||||
|
||||
async def scrape_catalog_list(
|
||||
page: Page,
|
||||
enseigne: Enseigne
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Scrape the list of catalogs for an ens enseigne from Bonial.
|
||||
|
||||
Args:
|
||||
page: Playwright page object
|
||||
enseigne: Enseigne object
|
||||
|
||||
Returns:
|
||||
List of catalog data dictionaries
|
||||
"""
|
||||
url = f"https://www.bonial.fr/Enseignes/{enseigne.slug_bonial}"
|
||||
logger.info(f"Scraping catalog list for {enseigne.nom} from {url}")
|
||||
|
||||
await page.goto(url, wait_until="networkidle")
|
||||
await accept_cookies(page)
|
||||
|
||||
# Wait for catalog cards to load
|
||||
try:
|
||||
await page.wait_for_selector(BONIAL_SELECTORS["catalog_card"], timeout=10000)
|
||||
except Exception as e:
|
||||
logger.error(f"No catalog cards found for {enseigne.nom}: {e}")
|
||||
return []
|
||||
|
||||
# Extract all catalog cards
|
||||
cards = await page.query_selector_all(BONIAL_SELECTORS["catalog_card"])
|
||||
logger.info(f"Found {len(cards)} catalog cards for {enseigne.nom}")
|
||||
|
||||
catalogues = []
|
||||
|
||||
for idx, card in enumerate(cards):
|
||||
try:
|
||||
# Extract title
|
||||
title_el = await card.query_selector(BONIAL_SELECTORS["catalog_title"])
|
||||
titre = await title_el.inner_text() if title_el else None
|
||||
|
||||
# Extract dates - look for any element containing date pattern
|
||||
dates_text = await card.inner_text()
|
||||
date_match = re.search(DATE_PATTERN, dates_text)
|
||||
|
||||
# Extract image
|
||||
image_el = await card.query_selector(BONIAL_SELECTORS["catalog_image"])
|
||||
image_url = await image_el.get_attribute("src") if image_el else None
|
||||
|
||||
# Extract link - try multiple strategies
|
||||
link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"])
|
||||
if not link_el:
|
||||
# Fallback: click on image
|
||||
link_el = image_el
|
||||
|
||||
catalogue_url = await link_el.get_attribute("href") if link_el else None
|
||||
|
||||
# Make URL absolute if relative
|
||||
if catalogue_url and not catalogue_url.startswith("http"):
|
||||
catalogue_url = f"https://www.bonial.fr{catalogue_url}"
|
||||
|
||||
if all([titre, date_match, image_url, catalogue_url]):
|
||||
# Parse dates
|
||||
try:
|
||||
date_debut, date_fin = parse_bonial_dates(dates_text)
|
||||
except ValueError as e:
|
||||
logger.warning(f"Failed to parse dates for catalog {idx}: {e}")
|
||||
continue
|
||||
|
||||
catalogues.append({
|
||||
"titre": titre.strip(),
|
||||
"date_debut": date_debut,
|
||||
"date_fin": date_fin,
|
||||
"image_couverture_url": image_url,
|
||||
"catalogue_url": catalogue_url,
|
||||
})
|
||||
|
||||
logger.debug(f"Extracted catalog: {titre}")
|
||||
else:
|
||||
logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={bool(date_match)}, image={bool(image_url)}, url={bool(catalogue_url)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Successfully extracted {len(catalogues)} catalogs for {enseigne.nom}")
|
||||
return catalogues
|
||||
|
||||
|
||||
async def scrape_catalog_pages(page: Page, catalogue_url: str) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Scrape all pages of a catalog from the Bonial viewer.
|
||||
|
||||
Args:
|
||||
page: Playwright page object
|
||||
catalogue_url: URL of the catalog viewer
|
||||
|
||||
Returns:
|
||||
List of page data dictionaries
|
||||
"""
|
||||
logger.info(f"Scraping catalog pages from {catalogue_url}")
|
||||
|
||||
await page.goto(catalogue_url, wait_until="networkidle")
|
||||
await asyncio.sleep(2) # Wait for viewer to initialize
|
||||
|
||||
# Try to close any popup/modal
|
||||
try:
|
||||
await page.click("button:has-text('Continuer'), button:has-text('Fermer')", timeout=2000)
|
||||
await asyncio.sleep(1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Find all images from content-media.bonial.biz
|
||||
images = await page.query_selector_all("img[src*='content-media.bonial.biz']")
|
||||
|
||||
if not images:
|
||||
logger.warning(f"No images found in catalog viewer: {catalogue_url}")
|
||||
return []
|
||||
|
||||
# Extract unique image URLs (filter duplicates and thumbnails)
|
||||
seen_urls = set()
|
||||
pages = []
|
||||
|
||||
for idx, img in enumerate(images):
|
||||
try:
|
||||
src = await img.get_attribute("src")
|
||||
|
||||
if src and src not in seen_urls:
|
||||
# Filter out blurred/low-res images if possible
|
||||
# Full res images are typically larger
|
||||
width = await img.evaluate("el => el.naturalWidth")
|
||||
height = await img.evaluate("el => el.naturalHeight")
|
||||
|
||||
# Skip very small images (likely thumbnails)
|
||||
if width < 200 or height < 200:
|
||||
continue
|
||||
|
||||
seen_urls.add(src)
|
||||
pages.append({
|
||||
"numero_page": len(pages) + 1,
|
||||
"image_url": src,
|
||||
"largeur": width,
|
||||
"hauteur": height,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error processing image {idx}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Found {len(pages)} pages in catalog")
|
||||
return pages
|
||||
|
||||
|
||||
async def scrape_enseigne(
|
||||
enseigne: Enseigne,
|
||||
db: Session,
|
||||
browser: Browser | None = None
|
||||
) -> ScrapingLog:
|
||||
"""
|
||||
Scrape all catalogs for a specific enseigne.
|
||||
|
||||
Args:
|
||||
enseigne: Enseigne to scrape
|
||||
db: Database session
|
||||
browser: Optional existing browser instance
|
||||
|
||||
Returns:
|
||||
ScrapingLog object with execution details
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
log = ScrapingLog(
|
||||
enseigne_id=enseigne.id,
|
||||
statut="error", # Will be updated
|
||||
catalogues_trouves=0,
|
||||
catalogues_nouveaux=0,
|
||||
catalogues_mis_a_jour=0,
|
||||
)
|
||||
|
||||
own_browser = browser is None
|
||||
|
||||
try:
|
||||
# Create browser if not provided
|
||||
if own_browser:
|
||||
p = await async_playwright().start()
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
|
||||
page = await browser.new_page()
|
||||
|
||||
# Set realistic user agent
|
||||
await page.set_extra_http_headers({
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
})
|
||||
|
||||
# Scrape catalog list
|
||||
catalogues_data = await scrape_catalog_list(page, enseigne)
|
||||
log.catalogues_trouves = len(catalogues_data)
|
||||
|
||||
if not catalogues_data:
|
||||
log.statut = "success"
|
||||
log.message_erreur = "No catalogs found (may be normal)"
|
||||
await page.close()
|
||||
return log
|
||||
|
||||
# Process each catalog
|
||||
for cat_data in catalogues_data:
|
||||
try:
|
||||
# Compute hash for duplicate detection
|
||||
content_hash = compute_catalog_hash(
|
||||
enseigne.id,
|
||||
cat_data["titre"],
|
||||
cat_data["date_debut"]
|
||||
)
|
||||
|
||||
# Check if catalog already exists
|
||||
existing = db.query(Catalogue).filter_by(content_hash=content_hash).first()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Catalog already exists: {cat_data['titre']}")
|
||||
continue
|
||||
|
||||
# Scrape catalog pages
|
||||
await asyncio.sleep(3) # Respectful delay
|
||||
pages_data = await scrape_catalog_pages(page, cat_data["catalogue_url"])
|
||||
|
||||
# Create new catalog
|
||||
catalogue = Catalogue(
|
||||
enseigne_id=enseigne.id,
|
||||
titre=cat_data["titre"],
|
||||
date_debut=cat_data["date_debut"],
|
||||
date_fin=cat_data["date_fin"],
|
||||
image_couverture_url=cat_data["image_couverture_url"],
|
||||
catalogue_url=cat_data["catalogue_url"],
|
||||
nombre_pages=len(pages_data),
|
||||
content_hash=content_hash,
|
||||
statut="actif" if cat_data["date_fin"] >= datetime.now() else "termine",
|
||||
)
|
||||
|
||||
db.add(catalogue)
|
||||
db.flush() # Get catalogue.id
|
||||
|
||||
# Add pages
|
||||
for page_data in pages_data:
|
||||
cat_page = CataloguePage(
|
||||
catalogue_id=catalogue.id,
|
||||
**page_data
|
||||
)
|
||||
db.add(cat_page)
|
||||
|
||||
log.catalogues_nouveaux += 1
|
||||
logger.info(f"Created catalog: {cat_data['titre']} ({len(pages_data)} pages)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing catalog {cat_data.get('titre')}: {e}")
|
||||
continue
|
||||
|
||||
await page.close()
|
||||
|
||||
# Update status
|
||||
log.statut = "success" if log.catalogues_nouveaux > 0 else "partial"
|
||||
db.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error scraping enseigne {enseigne.nom}: {e}")
|
||||
log.statut = "error"
|
||||
log.message_erreur = str(e)
|
||||
db.rollback()
|
||||
|
||||
finally:
|
||||
if own_browser and browser:
|
||||
await browser.close()
|
||||
|
||||
# Record duration
|
||||
log.duree_secondes = (datetime.now() - start_time).total_seconds()
|
||||
db.add(log)
|
||||
db.commit()
|
||||
|
||||
return log
|
||||
|
||||
|
||||
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
||||
"""
|
||||
Scrape all active enseignes.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of ScrapingLog objects
|
||||
"""
|
||||
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
|
||||
logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
|
||||
|
||||
logs = []
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
|
||||
for enseigne in enseignes:
|
||||
try:
|
||||
await asyncio.sleep(2) # Delay between enseignes
|
||||
log = await scrape_enseigne(enseigne, db, browser)
|
||||
logs.append(log)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to scrape {enseigne.nom}: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
logger.info(f"Scraping complete: {len(logs)} enseignes processed")
|
||||
return logs
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Scheduler for automated catalog scraping
|
||||
|
||||
Runs scraping jobs twice daily (6h and 18h) for all active enseignes.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.services.bonial_scraper import scrape_all_enseignes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Global scheduler instance
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
|
||||
async def scheduled_scrape_job():
|
||||
"""Scheduled job to scrape all active enseignes"""
|
||||
logger.info("Starting scheduled catalog scraping job")
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
logs = await scrape_all_ense ignes(db)
|
||||
|
||||
total_new = sum(log.catalogues_nouveaux for log in logs)
|
||||
total_found = sum(log.catalogues_trouves for log in logs)
|
||||
|
||||
logger.info(
|
||||
f"Scheduled scraping complete: "
|
||||
f"{len(logs)} enseignes processed, "
|
||||
f"{total_found} catalogues found, "
|
||||
f"{total_new} new catalogues created"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in scheduled scraping job: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def start_scheduler():
|
||||
"""Start the scheduler with configured jobs"""
|
||||
# Schedule scraping twice daily at 6:00 and 18:00
|
||||
scheduler.add_job(
|
||||
scheduled_scrape_job,
|
||||
CronTrigger(hour="6,18", minute=0),
|
||||
id="catalog_scraping",
|
||||
name="Scrape Bonial catalogs",
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
logger.info("Catalog scraping scheduler started (runs at 6:00 and 18:00)")
|
||||
|
||||
|
||||
def stop_scheduler():
|
||||
"""Stop the scheduler"""
|
||||
if scheduler.running:
|
||||
scheduler.shutdown()
|
||||
logger.info("Catalog scraping scheduler stopped")
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
Seed default enseignes (stores) for the catalog module
|
||||
"""
|
||||
|
||||
import logging
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Enseigne
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Data for the 9 enseignes
|
||||
ENSEIGNES_DATA = [
|
||||
{
|
||||
"nom": "Gifi",
|
||||
"slug_bonial": "Gifi",
|
||||
"couleur": "#E30613",
|
||||
"site_url": "https://www.gifi.fr",
|
||||
"description": "Décoration, maison, bazar",
|
||||
"ordre_affichage": 1,
|
||||
},
|
||||
{
|
||||
"nom": "Action",
|
||||
"slug_bonial": "Action",
|
||||
"couleur": "#0066B3",
|
||||
"site_url": "https://www.action.com/fr-fr/",
|
||||
"description": "Discount non-alimentaire",
|
||||
"ordre_affichage": 2,
|
||||
},
|
||||
{
|
||||
"nom": "Centrakor",
|
||||
"slug_bonial": "Centrakor",
|
||||
"couleur": "#E94E1B",
|
||||
"site_url": "https://www.centrakor.com",
|
||||
"description": "Décoration, maison",
|
||||
"ordre_affichage": 3,
|
||||
},
|
||||
{
|
||||
"nom": "La Foir'Fouille",
|
||||
"slug_bonial": "La-Foir-Fouille",
|
||||
"couleur": "#009639",
|
||||
"site_url": "https://www.lafoirfouille.fr",
|
||||
"description": "Bazar, décoration",
|
||||
"ordre_affichage": 4,
|
||||
},
|
||||
{
|
||||
"nom": "Stokomani",
|
||||
"slug_bonial": "Stokomani",
|
||||
"couleur": "#FF6600",
|
||||
"site_url": "https://www.stokomani.fr",
|
||||
"description": "Déstockage textile et maison",
|
||||
"ordre_affichage": 5,
|
||||
},
|
||||
{
|
||||
"nom": "B&M",
|
||||
"slug_bonial": "BM",
|
||||
"couleur": "#D4145A",
|
||||
"site_url": "https://bmstores.fr",
|
||||
"description": "Discount britannique",
|
||||
"ordre_affichage": 6,
|
||||
},
|
||||
{
|
||||
"nom": "L'Incroyable",
|
||||
"slug_bonial": "L-incroyable",
|
||||
"couleur": "#8B0000",
|
||||
"site_url": "https://www.lincroyable.fr",
|
||||
"description": "Décoration et mobilier discount (Groupe Althys, siège à Denain)",
|
||||
"ordre_affichage": 7,
|
||||
},
|
||||
{
|
||||
"nom": "Bazarland",
|
||||
"slug_bonial": "Bazarland",
|
||||
"couleur": "#FFCC00",
|
||||
"site_url": None,
|
||||
"description": "Bazar discount",
|
||||
"ordre_affichage": 8,
|
||||
},
|
||||
{
|
||||
"nom": "Noz",
|
||||
"slug_bonial": "Noz",
|
||||
"couleur": "#003366",
|
||||
"site_url": "https://www.noz.fr",
|
||||
"description": "Déstockage généraliste",
|
||||
"ordre_affichage": 9,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def seed_enseignes(db: Session) -> int:
|
||||
"""
|
||||
Seed the default enseignes if they don't exist.
|
||||
|
||||
Returns:
|
||||
Number of enseignes created
|
||||
"""
|
||||
created = 0
|
||||
|
||||
for data in ENSEIGNES_DATA:
|
||||
# Check if enseigne already exists
|
||||
existing = db.query(Enseigne).filter_by(slug_bonial=data["slug_bonial"]).first()
|
||||
|
||||
if not existing:
|
||||
enseigne = Enseigne(**data)
|
||||
db.add(enseigne)
|
||||
created += 1
|
||||
logger.info(f"Created enseigne: {data['nom']}")
|
||||
else:
|
||||
logger.debug(f"Enseigne already exists: {data['nom']}")
|
||||
|
||||
db.commit()
|
||||
logger.info(f"Seeding complete: {created} enseigne(s) created")
|
||||
|
||||
return created
|
||||
|
||||
|
||||
def get_enseigne_by_slug(db: Session, slug: str) -> Enseigne | None:
|
||||
"""Get an enseigne by its Bonial slug"""
|
||||
return db.query(Enseigne).filter_by(slug_bonial=slug).first()
|
||||
|
||||
|
||||
def get_all_active_enseignes(db: Session) -> list[Enseigne]:
|
||||
"""Get all active enseignes ordered by display order"""
|
||||
return (
|
||||
db.query(Enseigne)
|
||||
.filter_by(is_active=True)
|
||||
.order_by(Enseigne.ordre_affichage)
|
||||
.all()
|
||||
)
|
||||
@@ -0,0 +1,329 @@
|
||||
# API Documentation - Module Catalogues Bonial
|
||||
|
||||
## Vue d'ensemble
|
||||
|
||||
Le module Catalogues permet de consulter les catalogues promotionnels des enseignes discount scrapers depuis Bonial.fr.
|
||||
|
||||
**Base URL**: `/api/catalogues`
|
||||
|
||||
## Endpoints Publics
|
||||
|
||||
### 1. Liste des Enseignes
|
||||
|
||||
`GET /api/catalogues/enseignes`
|
||||
|
||||
Retourne la liste de toutes les enseignes actives avec le nombre de catalogues en cours.
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"nom": "Gifi",
|
||||
"slug_bonial": "Gifi",
|
||||
"logo_url": null,
|
||||
"couleur": "#E30613",
|
||||
"site_url": "https://www.gifi.fr",
|
||||
"description": "Décoration, maison, bazar",
|
||||
"is_active": true,
|
||||
"ordre_affichage": 1,
|
||||
"catalogues_actifs_count": 3
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. Liste des Catalogues (Paginée)
|
||||
|
||||
`GET /api/catalogues`
|
||||
|
||||
Retourne la liste paginée des catalogues avec filtres.
|
||||
|
||||
**Paramètres Query** :
|
||||
- `page` (int, default=1): Numéro de page
|
||||
- `limit` (int, default=20, max=100): Nombre d'éléments par page
|
||||
- `enseigne_ids` (string, optional): IDs d'enseignes séparés par virgule (ex: `1,2,3`)
|
||||
- `statut` (string, optional): Filtre par statut (`actif`, `termine`, `tous`)
|
||||
- `date_debut_min` (datetime, optional): Date de début minimum
|
||||
- `date_fin_max` (datetime, optional): Date de fin maximum
|
||||
- `recherche` (string, optional): Recherche dans les titres
|
||||
- `sort` (string, default=`date_debut`): Tri (`date_debut`, `date_fin`, `enseigne`)
|
||||
- `order` (string, default=`desc`): Ordre (`asc`, `desc`)
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": 42,
|
||||
"enseigne": {
|
||||
"id": 1,
|
||||
"nom": "Gifi",
|
||||
"couleur": "#E30613",
|
||||
...
|
||||
},
|
||||
"titre": "Catalogue Gifi Noël 2024",
|
||||
"date_debut": "2024-11-25T00:00:00",
|
||||
"date_fin": "2024-12-24T23:59:59",
|
||||
"image_couverture_url": "https://content-media.bonial.biz/...",
|
||||
"statut": "actif",
|
||||
"nombre_pages": 24,
|
||||
"created_at": "2024-11-29T06:00:00"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 156,
|
||||
"page": 1,
|
||||
"limit": 20,
|
||||
"pages_total": 8
|
||||
},
|
||||
"metadata": {
|
||||
"derniere_mise_a_jour": "2024-11-29T06:15:32"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. Détail d'un Catalogue
|
||||
|
||||
`GET /api/catalogues/{catalogue_id}`
|
||||
|
||||
Retourne les informations détaillées d'un catalogue.
|
||||
|
||||
**Paramètres Path** :
|
||||
- `catalogue_id` (int): ID du catalogue
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
{
|
||||
"id": 42,
|
||||
"enseigne": {
|
||||
"id": 1,
|
||||
"nom": "Gifi",
|
||||
...
|
||||
},
|
||||
"titre": "Catalogue Gifi Noël 2024",
|
||||
"description": null,
|
||||
"date_debut": "2024-11-25T00:00:00",
|
||||
"date_fin": "2024-12-24T23:59:59",
|
||||
"image_couverture_url": "https://content-media.bonial.biz/...",
|
||||
"catalogue_url": "https://www.bonial.fr/...",
|
||||
"statut": "actif",
|
||||
"nombre_pages": 24,
|
||||
"metadonnees": null,
|
||||
"created_at": "2024-11-29T06:00:00",
|
||||
"updated_at": "2024-11-29T06:00:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Erreurs** :
|
||||
- `404 Not Found`: Catalogue introuvable
|
||||
|
||||
---
|
||||
|
||||
### 4. Pages d'un Catalogue
|
||||
|
||||
`GET /api/catalogues/{catalogue_id}/pages`
|
||||
|
||||
Retourne toutes les pages d'un catalogue, ordonnées par numéro.
|
||||
|
||||
**Paramètres Path** :
|
||||
- `catalogue_id` (int): ID du catalogue
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 128,
|
||||
"numero_page": 1,
|
||||
"image_url": "https://content-media.bonial.biz/.../page_1.jpg",
|
||||
"image_thumbnail_url": null,
|
||||
"largeur": 1200,
|
||||
"hauteur": 1600
|
||||
},
|
||||
{
|
||||
"id": 129,
|
||||
"numero_page": 2,
|
||||
"image_url": "https://content-media.bonial.biz/.../page_2.jpg",
|
||||
"image_thumbnail_url": null,
|
||||
"largeur": 1200,
|
||||
"hauteur": 1600
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
**Erreurs** :
|
||||
- `404 Not Found`: Catalogue introuvable
|
||||
|
||||
---
|
||||
|
||||
## Endpoints Admin (Authentification requise)
|
||||
|
||||
### 5. Déclencher Scraping Manuel
|
||||
|
||||
`POST /api/catalogues/admin/scraping/trigger`
|
||||
|
||||
Déclenche un scraping manuel des catalogues Bonial.
|
||||
|
||||
**Authentification** : Requise (JWT token + rôle `admin`)
|
||||
|
||||
**Paramètres Query** :
|
||||
- `enseigne_id` (int, optional): ID de l'enseigne à scraper. Si omis, scrape toutes les enseignes actives.
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "Scraping complete for Gifi",
|
||||
"catalogues_trouves": 4,
|
||||
"catalogues_nouveaux": 2
|
||||
}
|
||||
```
|
||||
|
||||
Ou pour toutes les enseignes :
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message":" Scraping complete for all enseignes",
|
||||
"enseignes_processed": 9,
|
||||
"catalogues_trouves": 38,
|
||||
"catalogues_nouveaux": 12
|
||||
}
|
||||
```
|
||||
|
||||
**Erreurs** :
|
||||
- `403 Forbidden`: Accès admin requis
|
||||
- `404 Not Found`: Enseigne introuvable
|
||||
|
||||
---
|
||||
|
||||
### 6. Historique des Scraping
|
||||
|
||||
`GET /api/catalogues/admin/scraping/logs`
|
||||
|
||||
Retourne l'historique des exécutions de scraping.
|
||||
|
||||
**Authentification** : Requise (JWT token + rôle `admin`)
|
||||
|
||||
**Paramètres Query** :
|
||||
- `limit` (int, default=50, max=200): Nombre de logs à retourner
|
||||
- `enseigne_id` (int, optional): Filtre par enseigne
|
||||
- `statut` (string, optional): Filtre par statut (`success`, `error`, `partial`)
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 15,
|
||||
"date_execution": "2024-11-29T06:00:15",
|
||||
"enseigne_id": 1,
|
||||
"enseigne_nom": "Gifi",
|
||||
"statut": "success",
|
||||
"catalogues_trouves": 4,
|
||||
"catalogues_nouveaux": 2,
|
||||
"catalogues_mis_a_jour": 0,
|
||||
"duree_secondes": 45.2,
|
||||
"message_erreur": null
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 7. Statistiques de Scraping
|
||||
|
||||
`GET /api/catalogues/admin/stats`
|
||||
|
||||
Retourne les statistiques globales du module catalogues.
|
||||
|
||||
**Authentification** : Requise (JWT token + rôle `admin`)
|
||||
|
||||
**Réponse** : `200 OK`
|
||||
```json
|
||||
{
|
||||
"total_catalogues": 156,
|
||||
"catalogues_par_enseigne": {
|
||||
"Gifi": 18,
|
||||
"Action": 22,
|
||||
"Centrakor": 15,
|
||||
"La Foir'Fouille": 12,
|
||||
"Stokomani": 19,
|
||||
"B&M": 14,
|
||||
"L'Incroyable": 11,
|
||||
"Bazarland": 8,
|
||||
"Noz": 17
|
||||
},
|
||||
"derniere_mise_a_jour": "2024-11-29T06:15:32",
|
||||
"prochaine_execution": "Aujourd'hui à 18:00"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Erreurs Communes
|
||||
|
||||
### 400 Bad Request
|
||||
```json
|
||||
{
|
||||
"detail": "Invalid enseigne_ids format"
|
||||
}
|
||||
```
|
||||
|
||||
### 403 Forbidden
|
||||
```json
|
||||
{
|
||||
"detail": "Admin access required"
|
||||
}
|
||||
```
|
||||
|
||||
### 404 Not Found
|
||||
```json
|
||||
{
|
||||
"detail": "Catalogue not found"
|
||||
}
|
||||
```
|
||||
|
||||
### 500 Internal Server Error
|
||||
```json
|
||||
{
|
||||
"detail": "Internal server error"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Scheduler Automatique
|
||||
|
||||
Le scraping est exécuté automatiquement **2 fois par jour** :
|
||||
- **6h00** : Scraping matinal
|
||||
- **18h00** : Scraping du soir
|
||||
|
||||
Les catalogues sont automatiquement marqués comme `termine` lorsque leur `date_fin` est dépassée.
|
||||
|
||||
---
|
||||
|
||||
## Notes d'Implémentation
|
||||
|
||||
### Détection de Doublons
|
||||
|
||||
Le système utilise un hash SHA-256 basé sur :
|
||||
- `enseigne_id`
|
||||
- `titre`
|
||||
- `date_debut`
|
||||
|
||||
Cela évite de créer des doublons même si le catalogue est re-scrapé.
|
||||
|
||||
### Performance
|
||||
|
||||
- Pagination obligatoire (max 100 items/page)
|
||||
- Index DB sur `enseigne_id`, `date_debut`, `date_fin`, `statut`
|
||||
- Scraping asynchrone (n'impacte pas les performances API)
|
||||
|
||||
### Formats de Date
|
||||
|
||||
Toutes les dates sont en **ISO 8601** avec timezone UTC :
|
||||
```
|
||||
2024-11-29T06:15:32+00:00
|
||||
```
|
||||
@@ -5,6 +5,7 @@ import Layout from '@/components/layout/Layout'
|
||||
import Dashboard from '@/pages/Dashboard'
|
||||
import Search from '@/pages/Search'
|
||||
import MultiSearch from '@/pages/MultiSearch'
|
||||
import Catalogues from '@/pages/Catalogues'
|
||||
import Login from '@/pages/Login'
|
||||
import Admin from '@/pages/Admin'
|
||||
import { useTheme } from '@/hooks/use-theme'
|
||||
@@ -77,6 +78,7 @@ function AppRoutes() {
|
||||
<Route path="/" element={<Dashboard />} />
|
||||
<Route path="/search" element={<Search />} />
|
||||
<Route path="/compare" element={<MultiSearch />} />
|
||||
<Route path="/catalogues" element={<Catalogues />} />
|
||||
<Route
|
||||
path="/admin"
|
||||
element={
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import axios from 'axios';
|
||||
import { toast } from 'sonner';
|
||||
import { Loader2, Play, Clock, CheckCircle2, XCircle, AlertCircle } from 'lucide-react';
|
||||
|
||||
const API_BASE = '/api/catalogues/admin';
|
||||
|
||||
export default function CatalogueAdmin() {
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [scraping, setScraping] = useState(false);
|
||||
const [logs, setLogs] = useState([]);
|
||||
const [stats, setStats] = useState(null);
|
||||
const [enseignes, setEnseignes] = useState([]);
|
||||
|
||||
useEffect(() => {
|
||||
loadEnseignes();
|
||||
loadStats();
|
||||
loadLogs();
|
||||
}, []);
|
||||
|
||||
const loadEnseignes = async () => {
|
||||
try {
|
||||
const response = await axios.get('/api/catalogues/enseignes');
|
||||
setEnseignes(response.data);
|
||||
} catch (error) {
|
||||
console.error('Error loading enseignes:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const loadStats = async () => {
|
||||
try {
|
||||
const response = await axios.get(`${API_BASE}/stats`);
|
||||
setStats(response.data);
|
||||
} catch (error) {
|
||||
console.error('Error loading stats:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const loadLogs = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const response = await axios.get(`${API_BASE}/scraping/logs?limit=20`);
|
||||
setLogs(response.data);
|
||||
} catch (error) {
|
||||
console.error('Error loading logs:', error);
|
||||
toast.error('Erreur lors du chargement des logs');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const triggerScraping = async (enseigneId = null) => {
|
||||
setScraping(true);
|
||||
try {
|
||||
const params = enseigneId ? { enseigne_id: enseigneId } : {};
|
||||
const response = await axios.post(`${API_BASE}/scraping/trigger`, null, { params });
|
||||
|
||||
toast.success(response.data.message);
|
||||
toast.info(`${response.data.catalogues_nouveaux} nouveaux catalogues trouvés`);
|
||||
|
||||
// Reload data
|
||||
await Promise.all([loadStats(), loadLogs()]);
|
||||
} catch (error) {
|
||||
console.error('Error triggering scraping:', error);
|
||||
toast.error('Erreur lors du déclenchement du scraping');
|
||||
} finally {
|
||||
setScraping(false);
|
||||
}
|
||||
};
|
||||
|
||||
const getStatusIcon = (status) => {
|
||||
switch (status) {
|
||||
case 'success':
|
||||
return <CheckCircle2 className="h-5 w-5 text-green-500" />;
|
||||
case 'error':
|
||||
return <XCircle className="h-5 w-5 text-red-500" />;
|
||||
case 'partial':
|
||||
return <AlertCircle className="h-5 w-5 text-yellow-500" />;
|
||||
default:
|
||||
return <Clock className="h-5 w-5 text-gray-500" />;
|
||||
}
|
||||
};
|
||||
|
||||
const formatDate = (dateString) => {
|
||||
const date = new Date(dateString);
|
||||
return date.toLocaleString('fr-FR');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<h2 className="text-2xl font-bold mb-2">Administration Catalogues</h2>
|
||||
<p className="text-muted-foreground">Gérer le scraping des catalogues Bonial</p>
|
||||
</div>
|
||||
|
||||
{/* Stats Cards */}
|
||||
{stats && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<h3 className="text-sm font-medium text-muted-foreground mb-2">Total Catalogues</h3>
|
||||
<p className="text-3xl font-bold">{stats.total_catalogues}</p>
|
||||
</div>
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<h3 className="text-sm font-medium text-muted-foreground mb-2">Dernière Mise à Jour</h3>
|
||||
<p className="text-lg font-medium">
|
||||
{stats.derniere_mise_a_jour
|
||||
? formatDate(stats.derniere_mise_a_jour)
|
||||
: 'Jamais'}
|
||||
</p>
|
||||
</div>
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<h3 className="text-sm font-medium text-muted-foreground mb-2">Prochaine Exécution</h3>
|
||||
<p className="text-lg font-medium">{stats.prochaine_execution}</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Manual Scraping */}
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<h3 className="text-lg font-semibold mb-4">Scraping Manuel</h3>
|
||||
<div className="space-y-4">
|
||||
<button
|
||||
onClick={() => triggerScraping()}
|
||||
disabled={scraping}
|
||||
className="w-full md:w-auto px-6 py-3 bg-primary text-primary-foreground rounded-lg hover:bg-primary/90 disabled:opacity-50 flex items-center gap-2"
|
||||
>
|
||||
{scraping ? (
|
||||
<>
|
||||
<Loader2 className="h-5 w-5 animate-spin" />
|
||||
Scraping en cours...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Play className="h-5 w-5" />
|
||||
Scraper toutes les enseignes
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
|
||||
<div className="border-t pt-4">
|
||||
<p className="text-sm text-muted-foreground mb-3">Ou scraper une enseigne spécifique:</p>
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 gap-2">
|
||||
{enseignes.map(enseigne => (
|
||||
<button
|
||||
key={enseigne.id}
|
||||
onClick={() => triggerScraping(enseigne.id)}
|
||||
disabled={scraping}
|
||||
className="px-4 py-2 rounded-lg border hover:bg-secondary disabled:opacity-50 text-left"
|
||||
>
|
||||
<div className="font-medium">{enseigne.nom}</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{enseigne.catalogues_actifs_count} catalogue(s)
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Catalogues par Enseigne */}
|
||||
{stats && (
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<h3 className="text-lg font-semibold mb-4">Catalogues par Enseigne</h3>
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
{Object.entries(stats.catalogues_par_enseigne).map(([nom, count]) => (
|
||||
<div key={nom} className="flex justify-between items-center p-3 rounded-lg bg-secondary">
|
||||
<span className="font-medium">{nom}</span>
|
||||
<span className="text-lg font-bold text-primary">{count}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Scraping Logs */}
|
||||
<div className="bg-card rounded-lg border p-6">
|
||||
<div className="flex justify-between items-center mb-4">
|
||||
<h3 className="text-lg font-semibold">Historique des Scraping</h3>
|
||||
<button
|
||||
onClick={loadLogs}
|
||||
disabled={loading}
|
||||
className="px-3 py-1 text-sm rounded-lg border hover:bg-secondary"
|
||||
>
|
||||
Actualiser
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<div className="flex justify-center py-8">
|
||||
<Loader2 className="h-6 w-6 animate-spin text-primary" />
|
||||
</div>
|
||||
) : logs.length === 0 ? (
|
||||
<p className="text-center text-muted-foreground py-8">Aucun log disponible</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{logs.map(log => (
|
||||
<div
|
||||
key={log.id}
|
||||
className="flex items-start gap-3 p-4 rounded-lg border hover:bg-secondary/50 transition-colors"
|
||||
>
|
||||
<div className="mt-0.5">
|
||||
{getStatusIcon(log.statut)}
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="font-medium">{log.enseigne_nom || 'Toutes'}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatDate(log.date_execution)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="text-sm text-muted-foreground">
|
||||
{log.catalogues_trouves} trouvé(s) • {log.catalogues_nouveaux} nouveau(x)
|
||||
{log.duree_secondes && ` • ${log.duree_secondes.toFixed(1)}s`}
|
||||
</div>
|
||||
{log.message_erreur && (
|
||||
<div className="text-sm text-red-500 mt-1">
|
||||
{log.message_erreur}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Link, useLocation, useNavigate } from 'react-router-dom';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { LayoutDashboard, Search, TrendingUp, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react';
|
||||
import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet';
|
||||
import { cn } from '@/lib/utils';
|
||||
@@ -23,6 +23,7 @@ const Layout = ({ children, theme, toggleTheme }) => {
|
||||
{ icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' },
|
||||
{ icon: Search, label: t('nav.search') || 'Recherche', path: '/search' },
|
||||
{ icon: TrendingUp, label: 'Comparateur', path: '/compare' },
|
||||
{ icon: BookOpen, label: 'Catalogues', path: '/catalogues' },
|
||||
];
|
||||
|
||||
// Add admin link for admin users
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
Settings as SettingsIcon, Cpu, Clock, Bell, Globe,
|
||||
ChevronDown, ChevronUp, RefreshCw, RotateCcw, Edit2,
|
||||
Sparkles, Eye, Code, Brain, DollarSign, Filter,
|
||||
TrendingDown, CheckCircle2, Package, Bug, Download
|
||||
TrendingDown, CheckCircle2, Package, Bug, Download, BookOpen
|
||||
} from 'lucide-react';
|
||||
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
|
||||
import { Label } from '@/components/ui/label';
|
||||
@@ -18,6 +18,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@
|
||||
import { Slider } from '@/components/ui/slider';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { useAuth } from '@/hooks/use-auth';
|
||||
import CatalogueAdmin from '@/components/dashboard/CatalogueAdmin';
|
||||
|
||||
const API_URL = '/api';
|
||||
|
||||
@@ -495,6 +496,7 @@ export default function Admin() {
|
||||
{ id: 'sites', label: 'Sites', icon: Globe },
|
||||
{ id: 'jobs', label: 'Jobs', icon: Clock },
|
||||
{ id: 'notifications', label: 'Notifications', icon: Bell },
|
||||
{ id: 'catalogues', label: 'Catalogues', icon: BookOpen },
|
||||
{ id: 'debug', label: 'Debug', icon: Bug },
|
||||
];
|
||||
|
||||
@@ -1120,6 +1122,9 @@ export default function Admin() {
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
</div >
|
||||
|
||||
{/* Catalogues Tab */}
|
||||
{activeTab === 'catalogues' && <CatalogueAdmin />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye } from 'lucide-react';
|
||||
import axios from 'axios';
|
||||
|
||||
const API_BASE = '/api/catalogues';
|
||||
|
||||
export default function Catalogues() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [enseignes, setEnseignes] = useState([]);
|
||||
const [catalogues, setCatalogues] = useState([]);
|
||||
const [selectedEnseigne, setSelectedEnseigne] = useState(null);
|
||||
const [pagination, setPagination] = useState({ page: 1, limit: 12, total: 0 });
|
||||
const [selectedCatalogue, setSelectedCatalogue] = useState(null);
|
||||
const [cataloguePages, setCataloguePages] = useState([]);
|
||||
const [searchQuery, setSearchQuery] = useState('');
|
||||
|
||||
useEffect(() => {
|
||||
loadEnseignes();
|
||||
loadCatalogues();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
loadCatalogues();
|
||||
}, [pagination.page, selectedEnseigne, searchQuery]);
|
||||
|
||||
const loadEnseignes = async () => {
|
||||
try {
|
||||
const response = await axios.get(`${API_BASE}/enseignes`);
|
||||
setEnseignes(response.data);
|
||||
} catch (error) {
|
||||
console.error('Error loading enseignes:', error);
|
||||
toast.error('Erreur lors du chargement des enseignes');
|
||||
}
|
||||
};
|
||||
|
||||
const loadCatalogues = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const params = {
|
||||
page: pagination.page,
|
||||
limit: pagination.limit,
|
||||
statut: 'actif',
|
||||
};
|
||||
|
||||
if (selectedEnseigne) {
|
||||
params.enseigne_ids = selectedEnseigne.id.toString();
|
||||
}
|
||||
|
||||
if (searchQuery) {
|
||||
params.recherche = searchQuery;
|
||||
}
|
||||
|
||||
const response = await axios.get(API_BASE, { params });
|
||||
setCatalogues(response.data.data);
|
||||
setPagination(prev => ({ ...prev, total: response.data.pagination.total }));
|
||||
} catch (error) {
|
||||
console.error('Error loading catalogues:', error);
|
||||
toast.error('Erreur lors du chargement des catalogues');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const loadCataloguePages = async (catalogueId) => {
|
||||
try {
|
||||
const response = await axios.get(`${API_BASE}/${catalogueId}/pages`);
|
||||
setCataloguePages(response.data);
|
||||
} catch (error) {
|
||||
console.error('Error loading pages:', error);
|
||||
toast.error('Erreur lors du chargement des pages');
|
||||
}
|
||||
};
|
||||
|
||||
const openCatalogue = (catalogue) => {
|
||||
setSelectedCatalogue(catalogue);
|
||||
loadCataloguePages(catalogue.id);
|
||||
};
|
||||
|
||||
const closeCatalogue = () => {
|
||||
setSelectedCatalogue(null);
|
||||
setCataloguePages([]);
|
||||
};
|
||||
|
||||
const formatDate = (dateString) => {
|
||||
const date = new Date(dateString);
|
||||
return date.toLocaleDateString('fr-FR', { day: 'numeric', month: 'short', year: 'numeric' });
|
||||
};
|
||||
|
||||
const isValidNow = (catalogue) => {
|
||||
const now = new Date();
|
||||
const debut = new Date(catalogue.date_debut);
|
||||
const fin = new Date(catalogue.date_fin);
|
||||
return now >= debut && now <= fin;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="p-6 max-w-7xl mx-auto">
|
||||
{/* Header */}
|
||||
<div className="mb-8">
|
||||
<h1 className="text-3xl font-bold mb-2">Catalogues Promotionnels</h1>
|
||||
<p className="text-muted-foreground">
|
||||
Consultez les catalogues et prospectus des enseignes discount
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Search Bar */}
|
||||
<div className="mb-6">
|
||||
<div className="relative">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Rechercher dans les catalogues..."
|
||||
className="w-full pl-10 pr-4 py-3 rounded-lg border bg-background"
|
||||
value={searchQuery}
|
||||
onChange={(e) => {
|
||||
setSearchQuery(e.target.value);
|
||||
setPagination(prev => ({ ...prev, page: 1 }));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Enseignes Filter */}
|
||||
<div className="mb-6">
|
||||
<div className="flex gap-2 overflow-x-auto pb-2">
|
||||
<button
|
||||
onClick={() => {
|
||||
setSelectedEnseigne(null);
|
||||
setPagination(prev => ({ ...prev, page: 1 }));
|
||||
}}
|
||||
className={`px-4 py-2 rounded-lg whitespace-nowrap transition-colors ${!selectedEnseigne
|
||||
? 'bg-primary text-primary-foreground'
|
||||
: 'bg-secondary hover:bg-secondary/80'
|
||||
}`}
|
||||
>
|
||||
Toutes les enseignes
|
||||
</button>
|
||||
{enseignes.map(enseigne => (
|
||||
<button
|
||||
key={enseigne.id}
|
||||
onClick={() => {
|
||||
setSelectedEnseigne(enseigne);
|
||||
setPagination(prev => ({ ...prev, page: 1 }));
|
||||
}}
|
||||
className={`px-4 py-2 rounded-lg whitespace-nowrap transition-colors ${selectedEnseigne?.id === enseigne.id
|
||||
? 'bg-primary text-primary-foreground'
|
||||
: 'bg-secondary hover:bg-secondary/80'
|
||||
}`}
|
||||
style={
|
||||
selectedEnseigne?.id === enseigne.id
|
||||
? { backgroundColor: enseigne.couleur, color: 'white' }
|
||||
: {}
|
||||
}
|
||||
>
|
||||
{enseigne.nom}
|
||||
{enseigne.catalogues_actifs_count > 0 && (
|
||||
<span className="ml-2 px-2 py-0.5 rounded-full bg-white/20 text-xs">
|
||||
{enseigne.catalogues_actifs_count}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Catalogues Grid */}
|
||||
{loading ? (
|
||||
<div className="flex justify-center items-center py-20">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-primary" />
|
||||
</div>
|
||||
) : catalogues.length === 0 ? (
|
||||
<div className="text-center py-20">
|
||||
<BookOpen className="h-12 w-12 mx-auto mb-4 text-muted-foreground" />
|
||||
<h3 className="text-lg font-medium mb-2">Aucun catalogue trouvé</h3>
|
||||
<p className="text-muted-foreground">
|
||||
{searchQuery
|
||||
? 'Aucun catalogue ne correspond à votre recherche'
|
||||
: 'Aucun catalogue disponible pour le moment'}
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6">
|
||||
{catalogues.map(catalogue => (
|
||||
<div
|
||||
key={catalogue.id}
|
||||
className="group bg-card rounded-lg border overflow-hidden hover:shadow-lg transition-all cursor-pointer"
|
||||
onClick={() => openCatalogue(catalogue)}
|
||||
>
|
||||
{/* Cover Image */}
|
||||
<div className="relative aspect-[3/4] overflow-hidden bg-secondary">
|
||||
<img
|
||||
src={catalogue.image_couverture_url}
|
||||
alt={catalogue.titre}
|
||||
className="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
|
||||
/>
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-black/60 to-transparent opacity-0 group-hover:opacity-100 transition-opacity">
|
||||
<div className="absolute bottom-4 left-4 right-4 text-white">
|
||||
<Eye className="h-5 w-5 mx-auto mb-2" />
|
||||
<p className="text-sm text-center">Voir le catalogue</p>
|
||||
</div>
|
||||
</div>
|
||||
{isValidNow(catalogue) && (
|
||||
<div className="absolute top-2 right-2">
|
||||
<span className="px-2 py-1 rounded-full bg-green-500 text-white text-xs font-medium">
|
||||
En cours
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="p-4">
|
||||
{/* Enseigne Badge */}
|
||||
<div
|
||||
className="inline-block px-3 py-1 rounded-full text-white text-xs font-medium mb-2"
|
||||
style={{ backgroundColor: catalogue.enseigne.couleur }}
|
||||
>
|
||||
{catalogue.enseigne.nom}
|
||||
</div>
|
||||
|
||||
{/* Title */}
|
||||
<h3 className="font-semibold text-base mb-2 line-clamp-2">
|
||||
{catalogue.titre}
|
||||
</h3>
|
||||
|
||||
{/* Dates */}
|
||||
<div className="flex items-center gap-1 text-sm text-muted-foreground mb-2">
|
||||
<Calendar className="h-4 w-4" />
|
||||
<span>
|
||||
{formatDate(catalogue.date_debut)} - {formatDate(catalogue.date_fin)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Pages */}
|
||||
<div className="flex items-center gap-1 text-sm text-muted-foreground">
|
||||
<BookOpen className="h-4 w-4" />
|
||||
<span>{catalogue.nombre_pages} pages</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Pagination */}
|
||||
{pagination.total > pagination.limit && (
|
||||
<div className="mt-8 flex justify-center items-center gap-2">
|
||||
<button
|
||||
onClick={() => setPagination(prev => ({ ...prev, page: prev.page - 1 }))}
|
||||
disabled={pagination.page === 1}
|
||||
className="px-4 py-2 rounded-lg border bg-background disabled:opacity-50"
|
||||
>
|
||||
Précédent
|
||||
</button>
|
||||
<span className="px-4 py-2 text-sm text-muted-foreground">
|
||||
Page {pagination.page} sur {Math.ceil(pagination.total / pagination.limit)}
|
||||
</span>
|
||||
<button
|
||||
onClick={() => setPagination(prev => ({ ...prev, page: prev.page + 1 }))}
|
||||
disabled={pagination.page >= Math.ceil(pagination.total / pagination.limit)}
|
||||
className="px-4 py-2 rounded-lg border bg-background disabled:opacity-50"
|
||||
>
|
||||
Suivant
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Catalogue Viewer Modal */}
|
||||
{selectedCatalogue && (
|
||||
<div className="fixed inset-0 z-50 bg-black/80 flex items-center justify-center p-4">
|
||||
<div className="bg-background rounded-lg max-w-4xl w-full max-h-[90vh] flex flex-col">
|
||||
{/* Header */}
|
||||
<div className="p-4 border-b flex justify-between items-start">
|
||||
<div>
|
||||
<h2 className="text-xl font-bold mb-1">{selectedCatalogue.titre}</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{selectedCatalogue.enseigne.nom} • {cataloguePages.length} pages
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={closeCatalogue}
|
||||
className="px-4 py-2 rounded-lg hover:bg-secondary"
|
||||
>
|
||||
Fermer
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Pages Viewer */}
|
||||
<div className="flex-1 overflow-y-auto p-4">
|
||||
{cataloguePages.length === 0 ? (
|
||||
<div className="flex justify-center py-20">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-primary" />
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 gap-4">
|
||||
{cataloguePages.map(page => (
|
||||
<div
|
||||
key={page.id}
|
||||
className="relative aspect-[3/4] rounded-lg overflow-hidden border cursor-pointer hover:shadow-lg transition-shadow"
|
||||
onClick={() => window.open(page.image_url, '_blank')}
|
||||
>
|
||||
<img
|
||||
src={page.image_url}
|
||||
alt={`Page ${page.numero_page}`}
|
||||
className="w-full h-full object-cover"
|
||||
/>
|
||||
<div className="absolute top-2 left-2 px-2 py-1 rounded bg-black/70 text-white text-xs font-medium">
|
||||
Page {page.numero_page}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
# Setup and Migration Script for Bonial Module
|
||||
#
|
||||
# Run this script to set up the Bonial catalog module
|
||||
|
||||
Write-Host "================================================================" -ForegroundColor Cyan
|
||||
Write-Host " BONIAL CATALOG MODULE - SETUP SCRIPT" -ForegroundColor Cyan
|
||||
Write-Host "================================================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
|
||||
# Check if Docker is running
|
||||
Write-Host "[1/3] Checking Docker..." -ForegroundColor Yellow
|
||||
try {
|
||||
$dockerCheck = docker ps 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " ERROR: Docker is not running!" -ForegroundColor Red
|
||||
Write-Host " Please start Docker Desktop and try again." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
Write-Host " OK - Docker is running" -ForegroundColor Green
|
||||
} catch {
|
||||
Write-Host " ERROR: Docker not found!" -ForegroundColor Red
|
||||
Write-Host " Please install Docker Desktop." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run Alembic migration inside container
|
||||
Write-Host ""
|
||||
Write-Host "[2/3] Running database migration..." -ForegroundColor Yellow
|
||||
try {
|
||||
docker compose exec -T priceflow alembic upgrade head
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host " OK - Migration completed successfully" -ForegroundColor Green
|
||||
} else {
|
||||
Write-Host " WARNING: Migration may have failed" -ForegroundColor Yellow
|
||||
Write-Host " Tables may already exist - this is normal" -ForegroundColor Yellow
|
||||
}
|
||||
} catch {
|
||||
Write-Host " ERROR: Could not run migration" -ForegroundColor Red
|
||||
Write-Host " Error: $_" -ForegroundColor Red
|
||||
}
|
||||
|
||||
# Restart application to trigger seeding
|
||||
Write-Host ""
|
||||
Write-Host "[3/3] Restarting application to seed enseignes..." -ForegroundColor Yellow
|
||||
try {
|
||||
docker compose restart priceflow
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host " OK - Application restarted" -ForegroundColor Green
|
||||
Start-Sleep -Seconds 3
|
||||
Write-Host " Check logs for: 'X enseigne(s) créée(s)'" -ForegroundColor Cyan
|
||||
} else {
|
||||
Write-Host " ERROR: Could not restart application" -ForegroundColor Red
|
||||
}
|
||||
} catch {
|
||||
Write-Host " ERROR: $_" -ForegroundColor Red
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "================================================================" -ForegroundColor Cyan
|
||||
Write-Host " SETUP COMPLETE!" -ForegroundColor Green
|
||||
Write-Host "================================================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
Write-Host "Next steps:" -ForegroundColor Yellow
|
||||
Write-Host " 1. Check application logs:" -ForegroundColor White
|
||||
Write-Host " docker compose logs -f priceflow" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
Write-Host " 2. Run verification script:" -ForegroundColor White
|
||||
Write-Host " docker compose exec priceflow python verify_bonial.py" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
Write-Host " 3. Test API endpoints:" -ForegroundColor White
|
||||
Write-Host " Invoke-WebRequest http://localhost:8555/api/catalogues/enseignes" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
Verification Script for Bonial Catalog Module
|
||||
|
||||
Tests database setup, scraper functionality, and API endpoints.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
from app.database import SessionLocal, engine
|
||||
from app.models import Catalogue, CataloguePage, Enseigne, ScrapingLog
|
||||
from app.services.bonial_scraper import scrape_enseigne
|
||||
from app.services.seed_enseignes import seed_enseignes
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_BASE_URL = "http://localhost:8555/api"
|
||||
|
||||
|
||||
def check_database_tables():
|
||||
"""Verify that catalog module tables exist."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Phase 1: Checking Database Tables")
|
||||
logger.info("=" * 60)
|
||||
|
||||
inspector = inspect(engine)
|
||||
required_tables = [
|
||||
"enseignes",
|
||||
"catalogues",
|
||||
"catalogue_pages",
|
||||
"scraping_logs",
|
||||
]
|
||||
|
||||
existing_tables = inspector.get_table_names()
|
||||
|
||||
all_exist = True
|
||||
for table in required_tables:
|
||||
exists = table in existing_tables
|
||||
status = "✅" if exists else "❌"
|
||||
logger.info(f"{status} Table '{table}': {'EXISTS' if exists else 'MISSING'}")
|
||||
if not exists:
|
||||
all_exist = False
|
||||
|
||||
if all_exist:
|
||||
logger.info("\n✅ All required tables exist!\n")
|
||||
return True
|
||||
else:
|
||||
logger.error("\n❌ Some tables are missing. Run migration: alembic upgrade head\n")
|
||||
return False
|
||||
|
||||
|
||||
def check_enseignes_seeding():
|
||||
"""Verify that enseignes are seeded."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Phase 2: Checking Enseignes Seeding")
|
||||
logger.info("=" * 60)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
count = db.query(Enseigne).count()
|
||||
logger.info(f"Found {count} enseignes in database")
|
||||
|
||||
if count == 0:
|
||||
logger.info("Seeding enseignes...")
|
||||
created = seed_enseignes(db)
|
||||
logger.info(f"✅ Created {created} enseignes")
|
||||
count = created
|
||||
|
||||
if count >= 9:
|
||||
logger.info("\n✅ All 9 enseignes are seeded!\n")
|
||||
|
||||
# Display enseignes
|
||||
enseignes = db.query(Enseigne).order_by(Enseigne.ordre_affichage).all()
|
||||
for ens in enseignes:
|
||||
active = "✅" if ens.is_active else "⚠️"
|
||||
logger.info(f" {active} {ens.ordre_affichage}. {ens.nom} (slug: {ens.slug_bonial})")
|
||||
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"\n⚠️ Expected 9 enseignes, found {count}\n")
|
||||
return False
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def test_scraper():
|
||||
"""Test the Bonial scraper on one enseigne."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Phase 3: Testing Bonial Scraper (Gifi)")
|
||||
logger.info("=" * 60)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# Get Gifi enseigne
|
||||
gifi = db.query(Enseigne).filter_by(slug_bonial="Gifi").first()
|
||||
|
||||
if not gifi:
|
||||
logger.error("❌ Gifi enseigne not found")
|
||||
return False
|
||||
|
||||
logger.info(f"Testing scraper for: {gifi.nom}")
|
||||
logger.info("This may take 30-60 seconds...")
|
||||
|
||||
# Run scraper
|
||||
log = await scrape_enseigne(gifi, db)
|
||||
|
||||
# Display results
|
||||
logger.info(f"\nScraping completed:")
|
||||
logger.info(f" Status: {log.statut}")
|
||||
logger.info(f" Catalogues found: {log.catalogues_trouves}")
|
||||
logger.info(f" New catalogues: {log.catalogues_nouveaux}")
|
||||
logger.info(f" Duration: {log.duree_secondes:.2f}s")
|
||||
|
||||
if log.message_erreur:
|
||||
logger.warning(f" Error: {log.message_erreur}")
|
||||
|
||||
if log.statut in ["success", "partial"] and log.catalogues_trouves > 0:
|
||||
logger.info("\n✅ Scraper is working!\n")
|
||||
|
||||
# Display sample catalog
|
||||
cat = db.query(Catalogue).filter_by(enseigne_id=gifi.id).first()
|
||||
if cat:
|
||||
logger.info(f"Sample catalog:")
|
||||
logger.info(f" Titre: {cat.titre}")
|
||||
logger.info(f" Dates: {cat.date_debut.date()} → {cat.date_fin.date()}")
|
||||
logger.info(f" Pages: {cat.nombre_pages}")
|
||||
|
||||
return True
|
||||
else:
|
||||
logger.error("\n❌ Scraper failed or found no catalogs\n")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error testing scraper: {e}")
|
||||
return False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def test_api_endpoints():
|
||||
"""Test API endpoints."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Phase 4: Testing API Endpoints")
|
||||
logger.info("=" * 60)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
# Test 1: Get enseignes
|
||||
logger.info("\n1. Testing GET /api/catalogues/enseignes")
|
||||
try:
|
||||
response = await client.get(f"{API_BASE_URL}/catalogues/enseignes")
|
||||
if response.status_code == 200:
|
||||
enseignes = response.json()
|
||||
logger.info(f" ✅ Status 200 - Found {len(enseignes)} enseignes")
|
||||
if enseignes:
|
||||
logger.info(f" Sample: {enseignes[0]['nom']} ({enseignes[0]['catalogues_actifs_count']} catalogues)")
|
||||
else:
|
||||
logger.error(f" ❌ Status {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ Error: {e}")
|
||||
return False
|
||||
|
||||
# Test 2: Get catalogues
|
||||
logger.info("\n2. Testing GET /api/catalogues")
|
||||
try:
|
||||
response = await client.get(f"{API_BASE_URL}/catalogues?page=1&limit=5")
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
catalogues = data.get("data", [])
|
||||
pagination = data.get("pagination", {})
|
||||
logger.info(f" ✅ Status 200 - Found {pagination.get('total', 0)} catalogues")
|
||||
logger.info(f" Page: {pagination.get('page')}/{pagination.get('pages_total')}")
|
||||
if catalogues:
|
||||
logger.info(f" Sample: {catalogues[0]['titre']}")
|
||||
else:
|
||||
logger.error(f" ❌ Status {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ Error: {e}")
|
||||
return False
|
||||
|
||||
# Test 3: Get catalogue detail
|
||||
logger.info("\n3. Testing GET /api/catalogues/{id}")
|
||||
try:
|
||||
# Get first catalog ID
|
||||
db = SessionLocal()
|
||||
cat = db.query(Catalogue).first()
|
||||
db.close()
|
||||
|
||||
if cat:
|
||||
response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}")
|
||||
if response.status_code == 200:
|
||||
detail = response.json()
|
||||
logger.info(f" ✅ Status 200 - Catalogue: {detail['titre']}")
|
||||
logger.info(f" Pages: {detail['nombre_pages']}")
|
||||
else:
|
||||
logger.error(f" ❌ Status {response.status_code}")
|
||||
return False
|
||||
else:
|
||||
logger.warning(" ⚠️ No catalogues in DB to test detail endpoint")
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ Error: {e}")
|
||||
return False
|
||||
|
||||
# Test 4: Get catalogue pages
|
||||
logger.info("\n4. Testing GET /api/catalogues/{id}/pages")
|
||||
try:
|
||||
if cat:
|
||||
response = await client.get(f"{API_BASE_URL}/catalogues/{cat.id}/pages")
|
||||
if response.status_code == 200:
|
||||
pages = response.json()
|
||||
logger.info(f" ✅ Status 200 - Found {len(pages)} pages")
|
||||
if pages:
|
||||
logger.info(f" Sample page: {pages[0]['numero_page']} - {pages[0]['image_url'][:50]}...")
|
||||
else:
|
||||
logger.error(f" ❌ Status {response.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ Error: {e}")
|
||||
return False
|
||||
|
||||
# Test 5: Get stats (requires working DB)
|
||||
logger.info("\n5. Testing GET /api/catalogues/admin/stats")
|
||||
try:
|
||||
response = await client.get(f"{API_BASE_URL}/catalogues/admin/stats")
|
||||
if response.status_code == 200:
|
||||
stats = response.json()
|
||||
logger.info(f" ✅ Status 200 - Total catalogues: {stats['total_catalogues']}")
|
||||
logger.info(f" Prochaine exécution: {stats['prochaine_execution']}")
|
||||
else:
|
||||
logger.error(f" ❌ Status {response.status_code}")
|
||||
# Stats is optional, don't fail
|
||||
except Exception as e:
|
||||
logger.warning(f" ⚠️ Stats endpoint error (may require auth): {e}")
|
||||
|
||||
logger.info("\n✅ All API endpoints are working!\n")
|
||||
return True
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all verification tests."""
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("BONIAL CATALOG MODULE - VERIFICATION SCRIPT")
|
||||
logger.info("=" * 60 + "\n")
|
||||
|
||||
results = []
|
||||
|
||||
# Phase 1: Database tables
|
||||
results.append(("Database Tables", check_database_tables()))
|
||||
|
||||
if not results[0][1]:
|
||||
logger.error("\n❌ Database not ready. Please run: alembic upgrade head")
|
||||
sys.exit(1)
|
||||
|
||||
# Phase 2: Enseignes seeding
|
||||
results.append(("Enseignes Seeding", check_enseignes_seeding()))
|
||||
|
||||
# Phase 3: Scraper test (optional, can be slow)
|
||||
scraper_test = input("\nRun scraper test? (Gifi - takes ~60s) [y/N]: ").lower() == "y"
|
||||
if scraper_test:
|
||||
results.append(("Scraper Test", await test_scraper()))
|
||||
|
||||
# Phase 4: API endpoints (requires app to be running)
|
||||
api_test = input("\nTest API endpoints? (App must be running on :8555) [y/N]: ").lower() == "y"
|
||||
if api_test:
|
||||
results.append(("API Endpoints", await test_api_endpoints()))
|
||||
|
||||
# Summary
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("VERIFICATION SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
|
||||
for name, passed in results:
|
||||
status = "✅ PASSED" if passed else "❌ FAILED"
|
||||
logger.info(f"{status}: {name}")
|
||||
|
||||
all_passed = all(result[1] for result in results)
|
||||
|
||||
if all_passed:
|
||||
logger.info("\n✅ ALL TESTS PASSED - Bonial module is ready!")
|
||||
else:
|
||||
logger.error("\n❌ SOME TESTS FAILED - Please review errors above")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in new issue
Block a user