mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-21 22:28:11 +02:00
feat: Add Bonial.fr scraper service for promotional catalogs.
This commit is contained in:
1 parent
debc1a973f
commit
c8fcfd7a4d
1 file changed
+9
-1
@@ -371,6 +371,7 @@ async def scrape_enseigne(
|
|||||||
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
||||||
"""
|
"""
|
||||||
Scrape all active enseignes.
|
Scrape all active enseignes.
|
||||||
|
Uses Browserless service via WebSocket.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
db: Database session
|
db: Database session
|
||||||
@@ -378,13 +379,20 @@ async def scrape_all_enseignes(db: Session) -> list[ScrapingLog]:
|
|||||||
Returns:
|
Returns:
|
||||||
List of ScrapingLog objects
|
List of ScrapingLog objects
|
||||||
"""
|
"""
|
||||||
|
import os
|
||||||
|
|
||||||
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
|
enseignes = db.query(Enseigne).filter_by(is_active=True).all()
|
||||||
logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
|
logger.info(f"Starting scraping for {len(enseignes)} active enseignes")
|
||||||
|
|
||||||
logs = []
|
logs = []
|
||||||
|
|
||||||
|
# Get Browserless URL from environment
|
||||||
|
browserless_url = os.environ.get("BROWSERLESS_URL", "ws://browserless:3000")
|
||||||
|
logger.info(f"Connecting to Browserless at {browserless_url}")
|
||||||
|
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
browser = await p.chromium.launch(headless=True)
|
# Connect to Browserless instead of launching local browser
|
||||||
|
browser = await p.chromium.connect_over_cdp(browserless_url)
|
||||||
|
|
||||||
for enseigne in enseignes:
|
for enseigne in enseignes:
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in new issue
Block a user