Merge pull request #147 from R0m1k3/antigravity

feat: Implement Cataloguemate.fr scraper service, add catalogue route…
This commit is contained in:
LogiFlow authored and GitHub committed 2025-11-30 01:57:11 +01:00
commit fcb596211d
4 files changed
+289 -39

No files matched your search

+9 -12
View File
@@ -24,7 +24,8 @@ from app.schemas_catalogues import (
ScrapingLogResponse,
ScrapingStatsResponse,
)
from app.services.tiendeo_scraper import scrape_all_enseignes, scrape_enseigne
from app.services.cataloguemate_scraper import scrape_enseigne
# from app.services.tiendeo_scraper import scrape_all_enseignes, scrape_enseigne
logger = logging.getLogger(__name__)
@@ -258,25 +259,21 @@ async def trigger_scraping(
if not enseigne:
raise HTTPException(status_code=404, detail="Enseigne not found")
async with async_playwright() as p:
import os
browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000")
browser = await p.chromium.connect_over_cdp(browserless_url)
log = await scrape_enseigne(enseigne, db, browser)
await browser.close()
# Use Cataloguemate scraper
log = await scrape_enseigne(enseigne, db)
return {
"message": f"Scraping completed for {enseigne.nom}",
"catalogues_trouves": log.catalogues_trouves,
"catalogues_nouveaux": log.catalogues_nouveaux,
"status": log.statut,
"error": log.message_erreur
}
else:
logs = await scrape_all_enseignes(db)
total_nouveaux = sum(log.catalogues_nouveaux for log in logs)
# Not implemented yet for all enseignes in this version
return {
"message": f"Scraping completed for all enseignes",
"catalogues_nouveaux": total_nouveaux,
"message": "Scraping all enseignes not yet implemented with new scraper",
"catalogues_nouveaux": 0,
}