mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
Merge pull request #162 from R0m1k3/antigravity
feat: Add Cataloguemate.fr scraper to replace Tiendeo and introduce n…
This commit is contained in:
2 files changed
+168
-74
No files matched your search
@@ -102,58 +102,65 @@ async def scrape_catalog_list(enseigne: Enseigne) -> list[dict[str, Any]]:
|
||||
async def scrape_catalog_pages(catalog_url: str) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Scrape pages from a specific catalog.
|
||||
Detects total pages from pagination links on first page.
|
||||
Detects total pages from pagination links (must check page 2, as page 1 doesn't show them).
|
||||
"""
|
||||
pages = []
|
||||
page_num = 1
|
||||
max_pages = 100 # Default safety limit
|
||||
|
||||
# First page: detect total number of pages from pagination
|
||||
first_page_html, _ = await browserless_service.get_page_content(catalog_url)
|
||||
# IMPORTANT: Pagination links only appear on page 2+, not on page 1
|
||||
# So we fetch page 2 first to detect the max pages
|
||||
page_2_url = f"{catalog_url}?page=2"
|
||||
page_2_html, _ = await browserless_service.get_page_content(page_2_url)
|
||||
|
||||
if not first_page_html:
|
||||
logger.error(f"Failed to fetch first page for {catalog_url}")
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(first_page_html, 'html.parser')
|
||||
|
||||
# Try to detect max pages from pagination links
|
||||
# Look for links with ?page=X or just numeric text
|
||||
pagination_links = soup.find_all('a', href=True)
|
||||
page_numbers = []
|
||||
|
||||
for link in pagination_links:
|
||||
href = link.get('href', '')
|
||||
text = link.get_text(strip=True)
|
||||
if page_2_html:
|
||||
soup = BeautifulSoup(page_2_html, 'html.parser')
|
||||
|
||||
# Check if it's a page number link
|
||||
if 'page=' in href:
|
||||
try:
|
||||
page_match = re.search(r'page=(\d+)', href)
|
||||
if page_match:
|
||||
page_numbers.append(int(page_match.group(1)))
|
||||
except:
|
||||
pass
|
||||
elif text.isdigit():
|
||||
# Direct numeric link (e.g., "11")
|
||||
page_numbers.append(int(text))
|
||||
|
||||
if page_numbers:
|
||||
max_pages = max(page_numbers)
|
||||
logger.info(f"Detected {max_pages} total pages from pagination")
|
||||
# Try to detect max pages from pagination links on page 2
|
||||
pagination_links = soup.find_all('a', href=True)
|
||||
page_numbers = []
|
||||
|
||||
for link in pagination_links:
|
||||
href = link.get('href', '')
|
||||
text = link.get_text(strip=True)
|
||||
|
||||
# Check if it's a page number link
|
||||
if 'page=' in href:
|
||||
try:
|
||||
page_match = re.search(r'page=(\d+)', href)
|
||||
if page_match:
|
||||
page_numbers.append(int(page_match.group(1)))
|
||||
except:
|
||||
pass
|
||||
elif text.isdigit():
|
||||
# Direct numeric link (e.g., "11")
|
||||
page_numbers.append(int(text))
|
||||
|
||||
if page_numbers:
|
||||
detected_max = max(page_numbers)
|
||||
# Sanity check: cataloguemate sometimes shows wrong page counts (e.g., 188)
|
||||
# Real catalogs rarely exceed 30-40 pages. Cap at 50 to be safe.
|
||||
if detected_max > 50:
|
||||
logger.warning(f"Detected abnormally high page count ({detected_max}), capping at 50")
|
||||
max_pages = 50
|
||||
else:
|
||||
max_pages = detected_max
|
||||
logger.info(f"Detected {max_pages} total pages from pagination (checked page 2)")
|
||||
else:
|
||||
logger.warning(f"Could not detect page count from page 2, using default limit of {max_pages}")
|
||||
else:
|
||||
logger.warning(f"Could not detect page count, using default limit of {max_pages}")
|
||||
logger.warning(f"Could not fetch page 2 to detect pagination, using default limit")
|
||||
|
||||
# Now scrape all pages
|
||||
# Now scrape all pages starting from page 1
|
||||
while page_num <= max_pages:
|
||||
# Construct URL for specific page
|
||||
current_url = catalog_url if page_num == 1 else f"{catalog_url}?page={page_num}"
|
||||
|
||||
logger.info(f"Scraping page {page_num}/{max_pages}: {current_url}")
|
||||
|
||||
# Reuse first page HTML for page 1
|
||||
if page_num == 1:
|
||||
html_content = first_page_html
|
||||
# Reuse page 2 HTML if we're on page 2
|
||||
if page_num == 2 and page_2_html:
|
||||
html_content = page_2_html
|
||||
else:
|
||||
html_content, _ = await browserless_service.get_page_content(current_url)
|
||||
|
||||
@@ -252,12 +259,14 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog:
|
||||
log.catalogues_trouves = len(catalogs_list)
|
||||
|
||||
for cat_info in catalogs_list:
|
||||
# Check if catalog exists
|
||||
# Generate content hash BEFORE checking existence (need pages for first image)
|
||||
# First, check if catalog exists by URL
|
||||
existing = db.query(Catalogue).filter(
|
||||
Catalogue.catalogue_url == cat_info['url']
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
logger.info(f"Catalogue already exists by URL: {cat_info['title']} (ID: {existing.id})")
|
||||
continue
|
||||
|
||||
# 2. Scrape pages for new catalog
|
||||
@@ -265,12 +274,20 @@ async def scrape_enseigne(enseigne: Enseigne, db: Session) -> ScrapingLog:
|
||||
|
||||
if not pages:
|
||||
continue
|
||||
|
||||
# Create catalog entry
|
||||
# Generate content hash (URL + Title + Date)
|
||||
hash_input = f"{cat_info['url']}{cat_info['title']}{datetime.now().date()}".encode('utf-8')
|
||||
|
||||
# Generate content hash based on title and first image (stable identifier)
|
||||
hash_input = f"{cat_info['title']}{pages[0]['image_url']}".encode('utf-8')
|
||||
content_hash = hashlib.sha256(hash_input).hexdigest()
|
||||
|
||||
# Check if a catalog with the same content already exists
|
||||
existing_by_hash = db.query(Catalogue).filter(
|
||||
Catalogue.content_hash == content_hash
|
||||
).first()
|
||||
|
||||
if existing_by_hash:
|
||||
logger.info(f"Catalogue already exists by content hash: {cat_info['title']} (ID: {existing_by_hash.id}, original URL: {existing_by_hash.catalogue_url})")
|
||||
continue
|
||||
|
||||
new_cat = Catalogue(
|
||||
enseigne_id=enseigne.id,
|
||||
titre=cat_info['title'],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye, Trash2 } from 'lucide-react';
|
||||
import { Loader2, Calendar, BookOpen, Search, ChevronRight, Eye, Trash2, ChevronLeft, X } from 'lucide-react';
|
||||
import axios from 'axios';
|
||||
import { useAuth } from '../hooks/use-auth';
|
||||
|
||||
@@ -15,6 +15,7 @@ export default function Catalogues() {
|
||||
const [selectedCatalogue, setSelectedCatalogue] = useState(null);
|
||||
const [cataloguePages, setCataloguePages] = useState([]);
|
||||
const [searchQuery, setSearchQuery] = useState('');
|
||||
const [currentPageIndex, setCurrentPageIndex] = useState(0);
|
||||
const { user, getAuthHeaders } = useAuth();
|
||||
|
||||
useEffect(() => {
|
||||
@@ -86,12 +87,14 @@ export default function Catalogues() {
|
||||
|
||||
const openCatalogue = (catalogue) => {
|
||||
setSelectedCatalogue(catalogue);
|
||||
setCurrentPageIndex(0); // Reset to first page
|
||||
loadCataloguePages(catalogue.id);
|
||||
};
|
||||
|
||||
const closeCatalogue = () => {
|
||||
setSelectedCatalogue(null);
|
||||
setCataloguePages([]);
|
||||
setCurrentPageIndex(0);
|
||||
};
|
||||
|
||||
const formatDate = (dateString) => {
|
||||
@@ -126,6 +129,24 @@ export default function Catalogues() {
|
||||
}
|
||||
};
|
||||
|
||||
// Keyboard navigation support
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e) => {
|
||||
if (!selectedCatalogue) return;
|
||||
|
||||
if (e.key === 'ArrowLeft') {
|
||||
setCurrentPageIndex(prev => Math.max(0, prev - 1));
|
||||
} else if (e.key === 'ArrowRight') {
|
||||
setCurrentPageIndex(prev => Math.min(cataloguePages.length - 1, prev + 1));
|
||||
} else if (e.key === 'Escape') {
|
||||
closeCatalogue();
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('keydown', handleKeyDown);
|
||||
return () => window.removeEventListener('keydown', handleKeyDown);
|
||||
}, [selectedCatalogue, cataloguePages.length]);
|
||||
|
||||
return (
|
||||
<div className="p-6 max-w-7xl mx-auto">
|
||||
{/* Header */}
|
||||
@@ -311,53 +332,109 @@ export default function Catalogues() {
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Catalogue Viewer Modal */}
|
||||
{/* Catalogue Viewer Modal - Page by Page Navigation */}
|
||||
{selectedCatalogue && (
|
||||
<div className="fixed inset-0 z-50 bg-black/80 flex items-center justify-center p-4">
|
||||
<div className="bg-background rounded-lg max-w-4xl w-full max-h-[90vh] flex flex-col">
|
||||
<div
|
||||
className="fixed inset-0 z-50 bg-black/95 flex items-center justify-center"
|
||||
onClick={closeCatalogue}
|
||||
>
|
||||
<div
|
||||
className="relative w-full h-full flex flex-col"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="p-4 border-b flex justify-between items-start">
|
||||
<div>
|
||||
<h2 className="text-xl font-bold mb-1">{selectedCatalogue.titre}</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{selectedCatalogue.enseigne.nom} • {cataloguePages.length} pages
|
||||
</p>
|
||||
<div className="bg-black/50 backdrop-blur-sm p-4 flex justify-between items-center">
|
||||
<div className="flex items-center gap-4">
|
||||
<h2 className="text-white text-xl font-bold">{selectedCatalogue.titre}</h2>
|
||||
<span className="text-white/70 text-sm">
|
||||
{selectedCatalogue.enseigne.nom}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4">
|
||||
{cataloguePages.length > 0 && (
|
||||
<span className="text-white text-sm font-medium">
|
||||
Page {currentPageIndex + 1} / {cataloguePages.length}
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
onClick={closeCatalogue}
|
||||
className="p-2 text-white hover:bg-white/20 rounded-lg transition-colors"
|
||||
title="Fermer (ESC)"
|
||||
>
|
||||
<X className="h-6 w-6" />
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
onClick={closeCatalogue}
|
||||
className="px-4 py-2 rounded-lg hover:bg-secondary"
|
||||
>
|
||||
Fermer
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Pages Viewer */}
|
||||
<div className="flex-1 overflow-y-auto p-4">
|
||||
{/* Main Image Viewer */}
|
||||
<div className="flex-1 flex items-center justify-center p-8 relative">
|
||||
{cataloguePages.length === 0 ? (
|
||||
<div className="flex justify-center py-20">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-primary" />
|
||||
</div>
|
||||
<Loader2 className="h-12 w-12 animate-spin text-white" />
|
||||
) : (
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 gap-4">
|
||||
{cataloguePages.map(page => (
|
||||
<div
|
||||
<>
|
||||
{/* Current Page Image */}
|
||||
<div className="max-w-4xl max-h-full flex items-center justify-center">
|
||||
<img
|
||||
src={cataloguePages[currentPageIndex]?.image_url}
|
||||
alt={`Page ${currentPageIndex + 1}`}
|
||||
className="max-w-full max-h-full object-contain rounded-lg shadow-2xl"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Navigation Buttons */}
|
||||
{cataloguePages.length > 1 && (
|
||||
<>
|
||||
{/* Previous Button */}
|
||||
<button
|
||||
onClick={() => setCurrentPageIndex(prev => Math.max(0, prev - 1))}
|
||||
disabled={currentPageIndex === 0}
|
||||
className="absolute left-4 top-1/2 -translate-y-1/2 p-4 bg-black/50 hover:bg-black/70 text-white rounded-full disabled:opacity-30 disabled:cursor-not-allowed transition-all"
|
||||
title="Page précédente (←)"
|
||||
>
|
||||
<ChevronLeft className="h-8 w-8" />
|
||||
</button>
|
||||
|
||||
{/* Next Button */}
|
||||
<button
|
||||
onClick={() => setCurrentPageIndex(prev => Math.min(cataloguePages.length - 1, prev + 1))}
|
||||
disabled={currentPageIndex === cataloguePages.length - 1}
|
||||
className="absolute right-4 top-1/2 -translate-y-1/2 p-4 bg-black/50 hover:bg-black/70 text-white rounded-full disabled:opacity-30 disabled:cursor-not-allowed transition-all"
|
||||
title="Page suivante (→)"
|
||||
>
|
||||
<ChevronRight className="h-8 w-8" />
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Thumbnail Strip */}
|
||||
{cataloguePages.length > 1 && (
|
||||
<div className="bg-black/50 backdrop-blur-sm p-4">
|
||||
<div className="flex gap-2 overflow-x-auto pb-2">
|
||||
{cataloguePages.map((page, index) => (
|
||||
<button
|
||||
key={page.id}
|
||||
className="relative aspect-[3/4] rounded-lg overflow-hidden border cursor-pointer hover:shadow-lg transition-shadow"
|
||||
onClick={() => window.open(page.image_url, '_blank')}
|
||||
onClick={() => setCurrentPageIndex(index)}
|
||||
className={`relative flex-shrink-0 w-20 h-28 rounded-lg overflow-hidden border-2 transition-all ${index === currentPageIndex
|
||||
? 'border-primary scale-110'
|
||||
: 'border-white/20 hover:border-white/50'
|
||||
}`}
|
||||
>
|
||||
<img
|
||||
src={page.image_url}
|
||||
alt={`Page ${page.numero_page}`}
|
||||
alt={`Page ${index + 1}`}
|
||||
className="w-full h-full object-cover"
|
||||
/>
|
||||
<div className="absolute top-2 left-2 px-2 py-1 rounded bg-black/70 text-white text-xs font-medium">
|
||||
Page {page.numero_page}
|
||||
<div className="absolute bottom-0 left-0 right-0 bg-black/70 text-white text-xs text-center py-0.5">
|
||||
{index + 1}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
Reference in new issue
Block a user