From 99650a992bcdbefc137e6ee4cfc43bee55f80efd Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Sun, 30 Nov 2025 11:17:28 +0100 Subject: [PATCH] feat: Add Cataloguemate.fr scraper to replace Tiendeo and introduce new Catalogues frontend page. --- app/services/cataloguemate_scraper.py | 101 ++++++++++-------- frontend/src/pages/Catalogues.jsx | 141 ++++++++++++++++++++------ 2 files changed, 168 insertions(+), 74 deletions(-) diff --git a/app/services/cataloguemate_scraper.py b/app/services/cataloguemate_scraper.py index c43e89e..00fc51f 100644 --- a/app/services/cataloguemate_scraper.py +++ b/app/services/cataloguemate_scraper.py @@ -102,58 +102,65 @@ async def scrape_catalog_list(enseigne: Enseigne) -> list[dict[str, Any]]: async def scrape_catalog_pages(catalog_url: str) -> list[dict[str, Any]]: """ Scrape pages from a specific catalog. - Detects total pages from pagination links on first page. + Detects total pages from pagination links (must check page 2, as page 1 doesn't show them). """ pages = [] page_num = 1 max_pages = 100 # Default safety limit - # First page: detect total number of pages from pagination - first_page_html, _ = await browserless_service.get_page_content(catalog_url) + # IMPORTANT: Pagination links only appear on page 2+, not on page 1 + # So we fetch page 2 first to detect the max pages + page_2_url = f"{catalog_url}?page=2" + page_2_html, _ = await browserless_service.get_page_content(page_2_url) - if not first_page_html: - logger.error(f"Failed to fetch first page for {catalog_url}") - return [] - - soup = BeautifulSoup(first_page_html, 'html.parser') - - # Try to detect max pages from pagination links - # Look for links with ?page=X or just numeric text - pagination_links = soup.find_all('a', href=True) - page_numbers = [] - - for link in pagination_links: - href = link.get('href', '') - text = link.get_text(strip=True) + if page_2_html: + soup = BeautifulSoup(page_2_html, 'html.parser') - # Check if it's a page number link - if 'page=' in href: - try: - page_match = re.search(r'page=(\d+)', href) - if page_match: - page_numbers.append(int(page_match.group(1))) - except: - pass - elif text.isdigit(): - # Direct numeric link (e.g., "11") - page_numbers.append(int(text)) - - if page_numbers: - max_pages = max(page_numbers) - logger.info(f"Detected {max_pages} total pages from pagination") + # Try to detect max pages from pagination links on page 2 + pagination_links = soup.find_all('a', href=True) + page_numbers = [] + + for link in pagination_links: + href = link.get('href', '') + text = link.get_text(strip=True) + + # Check if it's a page number link + if 'page=' in href: + try: + page_match = re.search(r'page=(\d+)', href) + if page_match: + page_numbers.append(int(page_match.group(1))) + except: + pass + elif text.isdigit(): + # Direct numeric link (e.g., "11") + page_numbers.append(int(text)) + + if page_numbers: + detected_max = max(page_numbers) + # Sanity check: cataloguemate sometimes shows wrong page counts (e.g., 188) + # Real catalogs rarely exceed 30-40 pages. Cap at 50 to be safe. + if detected_max > 50: + logger.warning(f"Detected abnormally high page count ({detected_max}), capping at 50") + max_pages = 50 + else: + max_pages = detected_max + logger.info(f"Detected {max_pages} total pages from pagination (checked page 2)") + else: + logger.warning(f"Could not detect page count from page 2, using default limit of {max_pages}") else: - logger.warning(f"Could not detect page count, using default limit of {max_pages}") + logger.warning(f"Could not fetch page 2 to detect pagination, using default limit") - # Now scrape all pages + # Now scrape all pages starting from page 1 while page_num <= max_pages: # Construct URL for specific page current_url = catalog_url if page_num == 1 else f"{catalog_url}?page={page_num}" logger.info(f"Scraping page {page_num}/{max_pages}: {current_url}") - # Reuse first page HTML for page 1 - if page_num == 1: - html_content = first_page_html + # Reuse page 2 HTML if we're on page 2 + if page_num == 2 and page_2_html: + html_content = page_2_html else: html_content, _ = await browserless_service.get_page_content(current_url) @@ -252,12 +259,14 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog: log.catalogues_trouves = len(catalogs_list) for cat_info in catalogs_list: - # Check if catalog exists + # Generate content hash BEFORE checking existence (need pages for first image) + # First, check if catalog exists by URL existing = db.query(Catalogue).filter( Catalogue.catalogue_url == cat_info['url'] ).first() if existing: + logger.info(f"Catalogue already exists by URL: {cat_info['title']} (ID: {existing.id})") continue # 2. Scrape pages for new catalog @@ -265,12 +274,20 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog: if not pages: continue - - # Create catalog entry - # Generate content hash (URL + Title + Date) - hash_input = f"{cat_info['url']}{cat_info['title']}{datetime.now().date()}".encode('utf-8') + + # Generate content hash based on title and first image (stable identifier) + hash_input = f"{cat_info['title']}{pages[0]['image_url']}".encode('utf-8') content_hash = hashlib.sha256(hash_input).hexdigest() + # Check if a catalog with the same content already exists + existing_by_hash = db.query(Catalogue).filter( + Catalogue.content_hash == content_hash + ).first() + + if existing_by_hash: + logger.info(f"Catalogue already exists by content hash: {cat_info['title']} (ID: {existing_by_hash.id}, original URL: {existing_by_hash.catalogue_url})") + continue + new_cat = Catalogue( enseigne_id=enseigne.id, titre=cat_info['title'], diff --git a/frontend/src/pages/Catalogues.jsx b/frontend/src/pages/Catalogues.jsx index b7a240b..09c07fb 100644 --- a/frontend/src/pages/Catalogues.jsx +++ b/frontend/src/pages/Catalogues.jsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; import { toast } from 'sonner'; -import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye, Trash2 } from 'lucide-react'; +import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye, Trash2, ChevronLeft, X } from 'lucide-react'; import axios from 'axios'; import { useAuth } from '../hooks/use-auth'; @@ -15,6 +15,7 @@ export default function Catalogues() { const [selectedCatalogue, setSelectedCatalogue] = useState(null); const [cataloguePages, setCataloguePages] = useState([]); const [searchQuery, setSearchQuery] = useState(''); + const [currentPageIndex, setCurrentPageIndex] = useState(0); const { user, getAuthHeaders } = useAuth(); useEffect(() => { @@ -86,12 +87,14 @@ export default function Catalogues() { const openCatalogue = (catalogue) => { setSelectedCatalogue(catalogue); + setCurrentPageIndex(0); // Reset to first page loadCataloguePages(catalogue.id); }; const closeCatalogue = () => { setSelectedCatalogue(null); setCataloguePages([]); + setCurrentPageIndex(0); }; const formatDate = (dateString) => { @@ -126,6 +129,24 @@ export default function Catalogues() { } }; + // Keyboard navigation support + useEffect(() => { + const handleKeyDown = (e) => { + if (!selectedCatalogue) return; + + if (e.key === 'ArrowLeft') { + setCurrentPageIndex(prev => Math.max(0, prev - 1)); + } else if (e.key === 'ArrowRight') { + setCurrentPageIndex(prev => Math.min(cataloguePages.length - 1, prev + 1)); + } else if (e.key === 'Escape') { + closeCatalogue(); + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [selectedCatalogue, cataloguePages.length]); + return (
{/* Header */} @@ -311,53 +332,109 @@ export default function Catalogues() { )} - {/* Catalogue Viewer Modal */} + {/* Catalogue Viewer Modal - Page by Page Navigation */} {selectedCatalogue && ( -
-
+
+
e.stopPropagation()} + > {/* Header */} -
-
-

{selectedCatalogue.titre}

-

- {selectedCatalogue.enseigne.nom} • {cataloguePages.length} pages -

+
+
+

{selectedCatalogue.titre}

+ + {selectedCatalogue.enseigne.nom} + +
+
+ {cataloguePages.length > 0 && ( + + Page {currentPageIndex + 1} / {cataloguePages.length} + + )} +
-
- {/* Pages Viewer */} -
+ {/* Main Image Viewer */} +
{cataloguePages.length === 0 ? ( -
- -
+ ) : ( -
- {cataloguePages.map(page => ( -
+ {/* Current Page Image */} +
+ {`Page +
+ + {/* Navigation Buttons */} + {cataloguePages.length > 1 && ( + <> + {/* Previous Button */} + + + {/* Next Button */} + + + )} + + )} +
+ + {/* Thumbnail Strip */} + {cataloguePages.length > 1 && ( +
+
+ {cataloguePages.map((page, index) => ( + ))}
- )} -
+
+ )}
)}