mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
Merge pull request #122 from R0m1k3/antigravity
feat: implement API router for catalogue and enseigne management with…
This commit is contained in:
2 files changed
+50
-160
No files matched your search
@@ -167,166 +167,6 @@ async def get_catalogue_detail(
|
||||
if not catalogue:
|
||||
raise HTTPException(status_code=404, detail="Catalogue not found")
|
||||
|
||||
return CatalogueDetailResponse.model_validate(catalogue)
|
||||
|
||||
|
||||
@router.get("/{catalogue_id}/pages", response_model=list[CataloguePageResponse])
|
||||
async def get_catalogue_pages(
|
||||
catalogue_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get all pages of a catalogue"""
|
||||
# Verify catalogue exists
|
||||
catalogue = db.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
|
||||
|
||||
if not catalogue:
|
||||
raise HTTPException(status_code=404, detail="Catalogue not found")
|
||||
|
||||
# Get pages ordered by page number
|
||||
pages = (
|
||||
db.query(CataloguePage)
|
||||
.filter(CataloguePage.catalogue_id == catalogue_id)
|
||||
.order_by(CataloguePage.numero_page)
|
||||
.all()
|
||||
)
|
||||
|
||||
return [CataloguePageResponse.model_validate(page) for page in pages]
|
||||
|
||||
|
||||
# === Admin Endpoints ===
|
||||
|
||||
@router.post("/admin/scraping/trigger", response_model=dict)
|
||||
async def trigger_scraping(
|
||||
enseigne_id: int | None = Query(None, description="Enseigne ID to scrape (null = all)"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Manually trigger catalog scraping"""
|
||||
|
||||
if enseigne_id:
|
||||
# Scrape single enseigne
|
||||
enseigne = db.query(Enseigne).filter(Enseigne.id == enseigne_id).first()
|
||||
|
||||
if not enseigne:
|
||||
raise HTTPException(status_code=404, detail="Enseigne not found")
|
||||
|
||||
# Run scraping asynchronously (in background)
|
||||
import asyncio
|
||||
from app.services.bonial_scraper import scrape_enseigne as scrape_ens
|
||||
|
||||
log = await scrape_ens(enseigne, db)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Scraping complete for {enseigne.nom}",
|
||||
"catalogues_trouves": log.catalogues_trouves,
|
||||
"catalogues_nouveaux": log.catalogues_nouveaux,
|
||||
}
|
||||
else:
|
||||
# Scrape all enseignes
|
||||
logs = await scrape_all_enseignes(db)
|
||||
|
||||
total_new = sum(log.catalogues_nouveaux for log in logs)
|
||||
total_found = sum(log.catalogues_trouves for log in logs)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Scraping complete for all enseignes",
|
||||
"enseignes_processed": len(logs),
|
||||
"catalogues_trouves": total_found,
|
||||
"catalogues_nouveaux": total_new,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/admin/scraping/logs", response_model=list[ScrapingLogResponse])
|
||||
async def get_scraping_logs(
|
||||
enseigne_id: int | None = Query(None, description="Filter by enseigne"),
|
||||
statut: str | None = Query(None, description="Filter by status"),
|
||||
limit: int = Query(50, ge=1, le=200, description="Number of logs to return"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get scraping execution logs"""
|
||||
query = db.query(ScrapingLog)
|
||||
|
||||
if enseigne_id:
|
||||
query = query.filter(ScrapingLog.enseigne_id == enseigne_id)
|
||||
|
||||
if statut:
|
||||
query = query.filter(ScrapingLog.statut == statut)
|
||||
|
||||
logs = query.order_by(ScrapingLog.date_execution.desc()).limit(limit).all()
|
||||
|
||||
# Add enseigne name
|
||||
result = []
|
||||
for log in logs:
|
||||
log_dict = ScrapingLogResponse.model_validate(log).model_dump()
|
||||
if log.enseigne_id:
|
||||
enseigne = db.query(Enseigne).filter(Enseigne.id == log.enseigne_id).first()
|
||||
log_dict["enseigne_nom"] = enseigne.nom if enseigne else None
|
||||
else:
|
||||
log_dict["enseigne_nom"] = "Toutes"
|
||||
|
||||
result.append(ScrapingLogResponse(**log_dict))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/admin/stats", response_model=ScrapingStatsResponse)
|
||||
async def get_scraping_stats(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get scraping statistics"""
|
||||
|
||||
# Total catalogues
|
||||
total_catalogues = db.query(func.count(Catalogue.id)).scalar()
|
||||
|
||||
# Catalogues par enseigne
|
||||
catalogues_par_enseigne = {}
|
||||
enseignes = db.query(Enseigne).all()
|
||||
|
||||
for enseigne in enseignes:
|
||||
count = (
|
||||
db.query(func.count(Catalogue.id))
|
||||
.filter(Catalogue.enseigne_id == enseigne.id)
|
||||
.scalar()
|
||||
)
|
||||
catalogues_par_enseigne[enseigne.nom] = count
|
||||
|
||||
# Last scraping time
|
||||
last_log = (
|
||||
db.query(ScrapingLog)
|
||||
.order_by(ScrapingLog.date_execution.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
derniere_mise_a_jour = last_log.date_execution if last_log else None
|
||||
|
||||
# Next execution (6:00 or 18:00)
|
||||
now = datetime.now()
|
||||
if now.hour < 6:
|
||||
prochaine = f"Aujourd'hui à 6:00"
|
||||
elif now.hour < 18:
|
||||
prochaine = f"Aujourd'hui à 18:00"
|
||||
else:
|
||||
prochaine = f"Demain à 6:00"
|
||||
|
||||
return ScrapingStatsResponse(
|
||||
total_catalogues=total_catalogues,
|
||||
catalogues_par_enseigne=catalogues_par_enseigne,
|
||||
derniere_mise_a_jour=derniere_mise_a_jour,
|
||||
prochaine_execution=prochaine,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/admin/cleanup", response_model=dict)
|
||||
async def cleanup_catalogs(
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
):
|
||||
"""
|
||||
Delete invalid catalogs (0 pages or generic titles).
|
||||
Admin only.
|
||||
"""
|
||||
if not current_user.is_admin:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
|
||||
deleted_count = 0
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Script to manually clean up catalogs and trigger scraping
|
||||
# Usage: ./scripts/manual_cleanup_and_scrape.ps1
|
||||
|
||||
$BaseUrl = "http://localhost:8555/api"
|
||||
$Username = "admin"
|
||||
$Password = "admin"
|
||||
|
||||
Write-Host "1. Logging in as $Username..." -ForegroundColor Cyan
|
||||
try {
|
||||
$LoginBody = @{
|
||||
username = $Username
|
||||
password = $Password
|
||||
} | ConvertTo-Json
|
||||
|
||||
$LoginResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/auth/login" -Body $LoginBody -ContentType "application/json"
|
||||
$Token = $LoginResponse.token
|
||||
Write-Host " Success! Token received." -ForegroundColor Green
|
||||
}
|
||||
catch {
|
||||
Write-Host " Login failed. Please check if the server is running." -ForegroundColor Red
|
||||
Write-Error $_
|
||||
exit
|
||||
}
|
||||
|
||||
$Headers = @{
|
||||
Authorization = "Bearer $Token"
|
||||
}
|
||||
|
||||
Write-Host "`n2. Running Cleanup (Deleting invalid catalogs)..." -ForegroundColor Cyan
|
||||
try {
|
||||
$CleanupResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/catalogues/admin/cleanup" -Headers $Headers
|
||||
Write-Host " $($CleanupResponse.message)" -ForegroundColor Green
|
||||
}
|
||||
catch {
|
||||
Write-Host " Cleanup failed." -ForegroundColor Red
|
||||
Write-Error $_
|
||||
}
|
||||
|
||||
Write-Host "`n3. Triggering Scraping for Gifi (ID: 1)..." -ForegroundColor Cyan
|
||||
try {
|
||||
$ScrapeResponse = Invoke-RestMethod -Method Post -Uri "$BaseUrl/catalogues/admin/scraping/trigger?enseigne_id=1" -Headers $Headers
|
||||
Write-Host " $($ScrapeResponse.message)" -ForegroundColor Green
|
||||
Write-Host " Check logs at: $BaseUrl/catalogues/admin/scraping/logs" -ForegroundColor Yellow
|
||||
}
|
||||
catch {
|
||||
Write-Host " Scraping trigger failed." -ForegroundColor Red
|
||||
Write-Error $_
|
||||
}
|
||||
|
||||
Write-Host "`nDone." -ForegroundColor Cyan
|
||||
Reference in new issue
Block a user