mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: add Cataloguemate.fr scraper service for promotional catalogs.
This commit is contained in:
1 parent
69cfa4cfa9
commit
1176eec95a
1 file changed
+13
@@ -234,6 +234,19 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog:
|
||||
try:
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# 0. Test connection
|
||||
logger.info("Testing browser connection...")
|
||||
try:
|
||||
test_result = await crawler.arun(url="https://www.cataloguemate.fr/", config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
|
||||
if test_result.success:
|
||||
logger.info(f"Connection test successful (status: {len(test_result.html)} chars)")
|
||||
else:
|
||||
logger.error(f"Connection test failed: {test_result.error_message}")
|
||||
raise Exception(f"Connection failed: {test_result.error_message}")
|
||||
except Exception as e:
|
||||
logger.error(f"Browser/Network error: {e}")
|
||||
raise e
|
||||
|
||||
# 1. Get list of catalogs
|
||||
catalogs_list = await scrape_catalog_list(crawler, enseigne)
|
||||
log.catalogues_trouves = len(catalogs_list)
|
||||
|
||||
Reference in new issue
Block a user