From da90ea12f23a7c0aa0b75ae98442b326049fe80b Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sat, 29 Nov 2025 20:04:06 +0100 Subject: [PATCH] feat: implement Bonial.fr catalog scraper using Playwright --- app/services/bonial_scraper.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/services/bonial_scraper.py b/app/services/bonial_scraper.py index 2e8cbeb..01450da 100644 --- a/app/services/bonial_scraper.py +++ b/app/services/bonial_scraper.py @@ -179,12 +179,18 @@ async def scrape_catalog_list( # Strategy 1: The card itself is a link catalogue_url = await card.get_attribute("href") - # Strategy 2: Look for link inside + # Strategy 2: Look for specific catalog link inside if not catalogue_url: link_el = await card.query_selector(BONIAL_SELECTORS["catalog_link"]) if link_el: catalogue_url = await link_el.get_attribute("href") + # Strategy 3: Look for ANY link inside + if not catalogue_url: + link_el = await card.query_selector("a") + if link_el: + catalogue_url = await link_el.get_attribute("href") + # Make URL absolute if relative if catalogue_url and not catalogue_url.startswith("http"): catalogue_url = f"https://www.bonial.fr{catalogue_url}" @@ -215,7 +221,13 @@ async def scrape_catalog_list( }) logger.debug(f"Extracted catalog: {titre}") else: - logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={has_dates}, image={has_image}, url={has_url}") + # DEBUG: Log HTML content if extraction failed to help diagnosis + try: + html_content = await card.inner_html() + logger.warning(f"Incomplete data for catalog {idx}: title={titre}, dates={has_dates}, image={has_image}, url={has_url}") + logger.warning(f"Card HTML snippet: {html_content[:200]}...") + except Exception: + pass except Exception as e: logger.error(f"Error extracting catalog {idx} for {enseigne.nom}: {e}")