mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-21 22:28:11 +02:00
feat: Rebuild search system with modular, site-specific parsers
Complete refactoring of the search module for robustness and reliability: **New Architecture:** - Modular parser system with specialized parsers for each major site - Factory pattern for automatic parser selection - Generic fallback parser for sites without specific implementation **Specialized Parsers:** - ✅ Amazon.fr - Robust parser with multiple fallback selectors - Handles sponsored products, Prime eligibility, ratings, reviews - Based on proven amazon_scraper_v2.py implementation - ✅ Cdiscount.com - Optimized for Cdiscount's HTML structure - ✅ Fnac.com - Handles Fnac's picture elements and article structure - ✅ Darty.com - Specialized for electronics retailer - ✅ Boulanger.com - Optimized for Boulanger's product cards **Key Features:** - Multiple fallback selectors for each element (title, price, image) - Proper price parsing for French format (1 234,56 €) - Rating and review count extraction - Stock status detection - Image URL extraction with multiple attribute fallbacks - Query filtering to ensure relevant results **Benefits:** - Much more robust than previous generic parsing - Easier to maintain (one parser per site) - Easily extensible (add new parser for new site) - Better error handling and logging - Higher success rate for product extraction **Updated Services:** - improved_search_service.py now uses ParserFactory - Legacy _parse_results marked as deprecated - Conversion layer between ProductResult and SearchResult This implements the Amazon France model across all major sites for consistent, reliable product search results.
This commit is contained in:
10 files changed
+1257
-5
No files matched your search
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Generic Parser - Fallback parser for sites without specific implementation
|
||||
Uses configuration from search_config.py
|
||||
"""
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from .base_parser import BaseParser, ProductResult
|
||||
from app.core.search_config import SITE_CONFIGS
|
||||
|
||||
|
||||
class GenericParser(BaseParser):
|
||||
"""
|
||||
Generic parser using configured selectors from search_config.py
|
||||
|
||||
This parser is used as a fallback for sites that don't have
|
||||
a specialized parser implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, site_key: str):
|
||||
"""
|
||||
Initialize with site configuration
|
||||
|
||||
Args:
|
||||
site_key: Key in SITE_CONFIGS dict
|
||||
"""
|
||||
config = SITE_CONFIGS.get(site_key)
|
||||
if not config:
|
||||
raise ValueError(f"Unknown site key: {site_key}")
|
||||
|
||||
super().__init__(
|
||||
site_name=config.get("name", site_key),
|
||||
base_url=config.get("search_url", "").split("/")[0:3] # Extract base URL
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.site_key = site_key
|
||||
|
||||
# Extract base_url properly
|
||||
if "search_url" in config:
|
||||
parts = config["search_url"].split("/")
|
||||
if len(parts) >= 3:
|
||||
self.base_url = f"{parts[0]}//{parts[2]}"
|
||||
|
||||
def parse_search_results(self, html: str, query: str, search_url: str) -> list[ProductResult]:
|
||||
"""
|
||||
Parse search results using configured selectors
|
||||
|
||||
Uses:
|
||||
- product_selector: CSS selector for product links
|
||||
- product_image_selector: CSS selector for images (optional)
|
||||
"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
products = []
|
||||
|
||||
# Get product links using configured selector
|
||||
product_selector = self.config.get("product_selector")
|
||||
if not product_selector:
|
||||
self.logger.error(f"No product_selector configured for {self.site_key}")
|
||||
return []
|
||||
|
||||
links = soup.select(product_selector)
|
||||
|
||||
if not links:
|
||||
self.logger.warning(f"No products found for {self.site_key} with selector: {product_selector}")
|
||||
return []
|
||||
|
||||
self.logger.info(f"Found {len(links)} product links for {self.site_name}")
|
||||
|
||||
seen_urls = set()
|
||||
|
||||
for link in links:
|
||||
try:
|
||||
href = link.get("href")
|
||||
if not href:
|
||||
continue
|
||||
|
||||
full_url = urljoin(search_url, href)
|
||||
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
# Extract title
|
||||
title = link.get_text(strip=True)
|
||||
|
||||
# Fallback: try title attribute or alt in img
|
||||
if not title:
|
||||
title = link.get("title", "")
|
||||
if not title:
|
||||
img = link.find("img")
|
||||
if img:
|
||||
title = img.get("alt", "")
|
||||
|
||||
if not title or len(title) < 3:
|
||||
continue
|
||||
|
||||
# Filter by query (relaxed: at least one word match)
|
||||
if not self.filter_by_query(title, query, strict=False):
|
||||
continue
|
||||
|
||||
# Extract image using configured selector
|
||||
image_url = None
|
||||
image_selector = self.config.get("product_image_selector")
|
||||
|
||||
if image_selector:
|
||||
# Try in link first
|
||||
img_elem = link.select_one(image_selector)
|
||||
|
||||
# Try in parent container
|
||||
if not img_elem:
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
img_elem = parent.select_one(image_selector)
|
||||
|
||||
if img_elem:
|
||||
image_url = self._get_image_src(img_elem)
|
||||
|
||||
# Fallback: any img in link or parent
|
||||
if not image_url:
|
||||
img = link.find("img")
|
||||
if img:
|
||||
image_url = self._get_image_src(img)
|
||||
|
||||
if not image_url:
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
img = parent.find("img")
|
||||
if img:
|
||||
image_url = self._get_image_src(img)
|
||||
|
||||
# Make image URL absolute
|
||||
if image_url:
|
||||
# Skip invalid images
|
||||
if image_url.startswith("data:") or "1x1" in image_url or "placeholder" in image_url.lower():
|
||||
image_url = None
|
||||
else:
|
||||
image_url = urljoin(search_url, image_url)
|
||||
|
||||
products.append(ProductResult(
|
||||
title=title,
|
||||
url=full_url,
|
||||
source=self.site_name,
|
||||
image_url=image_url,
|
||||
currency="EUR",
|
||||
snippet=f"Product from {self.site_name}"
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error parsing product from {self.site_key}: {e}")
|
||||
continue
|
||||
|
||||
self.logger.info(f"Extracted {len(products)} products from {self.site_name}")
|
||||
return products
|
||||
Reference in new issue
Block a user