mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: Add Bonial.fr scraper service for promotional catalogs.
This commit is contained in:
1 parent
debc1a973f
commit
c8fcfd7a4d
1 file changed
+9
-1
@@ -371,6 +371,7 @@ async def scrape_enseigne(
|
||||
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
||||
"""
|
||||
Scrape all active enseignes.
|
||||
Uses Browserless service via WebSocket.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
@@ -378,13 +379,20 @@ async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
||||
Returns:
|
||||
List of ScrapingLog objects
|
||||
"""
|
||||
import os
|
||||
|
||||
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
|
||||
logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
|
||||
|
||||
logs = []
|
||||
|
||||
# Get Browserless URL from environment
|
||||
browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000")
|
||||
logger.info(f"Connecting to Browserless at {browserless_url}")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
# Connect to Browserless instead of launching local browser
|
||||
browser = await p.chromium.connect_over_cdp(browserless_url)
|
||||
|
||||
for enseigne in enseignes:
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user