From b224c4988ae64955b82520a7441fed8f42a05bfb Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sat, 29 Nov 2025 22:39:34 +0100 Subject: [PATCH] feat: implement API router for catalogue and enseigne management with filtering and pagination --- app/routers/catalogues.py | 160 -------------------------- scripts/manual_cleanup_and_scrape.ps1 | 50 ++++++++ 2 files changed, 50 insertions(+), 160 deletions(-) create mode 100644 scripts/manual_cleanup_and_scrape.ps1 diff --git a/app/routers/catalogues.py b/app/routers/catalogues.py index 041fa60..9a009bd 100644 --- a/app/routers/catalogues.py +++ b/app/routers/catalogues.py @@ -167,166 +167,6 @@ async def get_catalogue_detail( if not catalogue: raise HTTPException(status_code=404, detail="Catalogue not found") - return CatalogueDetailResponse.model_validate(catalogue) - - -@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse]) -async def get_catalogue_pages( - catalogue_id: int, - db: Session = Depends(get_db), -): - """Get all pages of a catalogue""" - # Verify catalogue exists - catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first() - - if not catalogue: - raise HTTPException(status_code=404, detail="Catalogue not found") - - # Get pages ordered by page number - pages = ( - db.query(CataloguePage) - .filter(CataloguePage.catalogue_id == catalogue_id) - .order_by(CataloguePage.numero_page) - .all() - ) - - return [CataloguePageResponse.model_validate(page) for page in pages] - - -# === Admin Endpoints === - -@router.post("/admin/scraping/trigger", response_model=dict) -async def trigger_scraping( - enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"), - db: Session = Depends(get_db), -): - """Manually trigger catalog scraping""" - - if enseigne_id: - # Scrape single enseigne - enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first() - - if not enseigne: - raise HTTPException(status_code=404, detail="Enseigne not found") - - # Run scraping asynchronously (in background) - import asyncio - from app.services.bonial_scraper import scrape_enseigne as scrape_ens - - log = await scrape_ens(enseigne, db) - - return { - "success": True, - "message": f"Scraping complete for {enseigne.nom}", - "catalogues_trouves": log.catalogues_trouves, - "catalogues_nouveaux": log.catalogues_nouveaux, - } - else: - # Scrape all enseignes - logs = await scrape_all_enseignes(db) - - total_new = sum(log.catalogues_nouveaux for log in logs) - total_found = sum(log.catalogues_trouves for log in logs) - - return { - "success": True, - "message": f"Scraping complete for all enseignes", - "enseignes_processed": len(logs), - "catalogues_trouves": total_found, - "catalogues_nouveaux": total_new, - } - - -@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse]) -async def get_scraping_logs( - enseigne_id: int | None = Query(None, description="Filter by enseigne"), - statut: str | None = Query(None, description="Filter by status"), - limit: int = Query(50, ge=1, le=200, description="Number of logs to return"), - db: Session = Depends(get_db), -): - """Get scraping execution logs""" - query = db.query(ScrapingLog) - - if enseigne_id: - query = query.filter(ScrapingLog.enseigne_id == enseigne_id) - - if statut: - query = query.filter(ScrapingLog.statut == statut) - - logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all() - - # Add enseigne name - result = [] - for log in logs: - log_dict = ScrapingLogResponse.model_validate(log).model_dump() - if log.enseigne_id: - enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first() - log_dict["enseigne_nom"] = enseigne.nom if enseigne else None - else: - log_dict["enseigne_nom"] = "Toutes" - - result.append(ScrapingLogResponse(**log_dict)) - - return result - - -@router.get("/admin/stats", response_model=ScrapingStatsResponse) -async def get_scraping_stats( - db: Session = Depends(get_db), -): - """Get scraping statistics""" - - # Total catalogues - total_catalogues = db.query(func.count(Catalogue.id)).scalar() - - # Catalogues par enseigne - catalogues_par_enseigne = {} - enseignes = db.query(Enseigne).all() - - for enseigne in enseignes: - count = ( - db.query(func.count(Catalogue.id)) - .filter(Catalogue.enseigne_id == enseigne.id) - .scalar() - ) - catalogues_par_enseigne[enseigne.nom] = count - - # Last scraping time - last_log = ( - db.query(ScrapingLog) - .order_by(ScrapingLog.date_execution.desc()) - .first() - ) - - derniere_mise_a_jour = last_log.date_execution if last_log else None - - # Next execution (6:00 or 18:00) - now = datetime.now() - if now.hour < 6: - prochaine = f"Aujourd'hui à 6:00" - elif now.hour < 18: - prochaine = f"Aujourd'hui à 18:00" - else: - prochaine = f"Demain à 6:00" - - return ScrapingStatsResponse( - total_catalogues=total_catalogues, - catalogues_par_enseigne=catalogues_par_enseigne, - derniere_mise_a_jour=derniere_mise_a_jour, - prochaine_execution=prochaine, - ) - - -@router.post("/admin/cleanup", response_model=dict) -async def cleanup_catalogs( - db: Session = Depends(get_db), - current_user: User = Depends(get_current_active_user), -): - """ - Delete invalid catalogs (0 pages or generic titles). - Admin only. - """ - if not current_user.is_admin: raise HTTPException(status_code=403, detail="Not authorized") deleted_count = 0 diff --git a/scripts/manual_cleanup_and_scrape.ps1 b/scripts/manual_cleanup_and_scrape.ps1 new file mode 100644 index 0000000..7be7ec0 --- /dev/null +++ b/scripts/manual_cleanup_and_scrape.ps1 @@ -0,0 +1,50 @@ +# Script to manually clean up catalogs and trigger scraping +# Usage: ./scripts/manual_cleanup_and_scrape.ps1 + +$BaseUrl = "http://localhost:8555/api" +$Username = "admin" +$Password = "admin" + +Write-Host "1. Logging in as $Username..." -ForegroundColor Cyan +try { + $LoginBody = @{ + username = $Username + password = $Password + } | ConvertTo-Json + + $LoginResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/auth/login" -Body $LoginBody -ContentType "application/json" + $Token = $LoginResponse.token + Write-Host " Success! Token received." -ForegroundColor Green +} +catch { + Write-Host " Login failed. Please check if the server is running." -ForegroundColor Red + Write-Error $_ + exit +} + +$Headers = @{ + Authorization = "Bearer $Token" +} + +Write-Host "`n2. Running Cleanup (Deleting invalid catalogs)..." -ForegroundColor Cyan +try { + $CleanupResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/catalogues/admin/cleanup" -Headers $Headers + Write-Host " $($CleanupResponse.message)" -ForegroundColor Green +} +catch { + Write-Host " Cleanup failed." -ForegroundColor Red + Write-Error $_ +} + +Write-Host "`n3. Triggering Scraping for Gifi (ID: 1)..." -ForegroundColor Cyan +try { + $ScrapeResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/catalogues/admin/scraping/trigger?enseigne_id=1" -Headers $Headers + Write-Host " $($ScrapeResponse.message)" -ForegroundColor Green + Write-Host " Check logs at: $BaseUrl/catalogues/admin/scraping/logs" -ForegroundColor Yellow +} +catch { + Write-Host " Scraping trigger failed." -ForegroundColor Red + Write-Error $_ +} + +Write-Host "`nDone." -ForegroundColor Cyan