mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-22 06:38:12 +02:00
feat: Rebuild search system with modular, site-specific parsers
Complete refactoring of the search module for robustness and reliability: **New Architecture:** - Modular parser system with specialized parsers for each major site - Factory pattern for automatic parser selection - Generic fallback parser for sites without specific implementation **Specialized Parsers:** - ✅ Amazon.fr - Robust parser with multiple fallback selectors - Handles sponsored products, Prime eligibility, ratings, reviews - Based on proven amazon_scraper_v2.py implementation - ✅ Cdiscount.com - Optimized for Cdiscount's HTML structure - ✅ Fnac.com - Handles Fnac's picture elements and article structure - ✅ Darty.com - Specialized for electronics retailer - ✅ Boulanger.com - Optimized for Boulanger's product cards **Key Features:** - Multiple fallback selectors for each element (title, price, image) - Proper price parsing for French format (1 234,56 €) - Rating and review count extraction - Stock status detection - Image URL extraction with multiple attribute fallbacks - Query filtering to ensure relevant results **Benefits:** - Much more robust than previous generic parsing - Easier to maintain (one parser per site) - Easily extensible (add new parser for new site) - Better error handling and logging - Higher success rate for product extraction **Updated Services:** - improved_search_service.py now uses ParserFactory - Legacy _parse_results marked as deprecated - Conversion layer between ProductResult and SearchResult This implements the Amazon France model across all major sites for consistent, reliable product search results.
This commit is contained in:
10 files changed
+1257
-5
No files matched your search
@@ -0,0 +1,131 @@
|
||||
"""
|
||||
Darty Parser - Specialized parser for Darty.com
|
||||
"""
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base_parser import BaseParser, ProductResult
|
||||
|
||||
|
||||
class DartyParser(BaseParser):
|
||||
"""
|
||||
Parser for Darty.com
|
||||
|
||||
Key selectors:
|
||||
- Product links: a[href*='/nav/achat/'][href*='.html']
|
||||
- Images: img.product_img
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
site_name="Darty",
|
||||
base_url="https://www.darty.com"
|
||||
)
|
||||
|
||||
def parse_search_results(self, html: str, query: str, search_url: str) -> list[ProductResult]:
|
||||
"""Parse Darty search results"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
products = []
|
||||
|
||||
# Try multiple selectors
|
||||
selectors = [
|
||||
"a[href*='/nav/achat/'][href*='.html']",
|
||||
".product-card a",
|
||||
"[class*='product'] a[href*='/nav/achat/']",
|
||||
]
|
||||
|
||||
links = []
|
||||
for selector in selectors:
|
||||
links = soup.select(selector)
|
||||
if links:
|
||||
self.logger.info(f"Found {len(links)} products with selector: {selector}")
|
||||
break
|
||||
|
||||
if not links:
|
||||
self.logger.warning("No products found on Darty")
|
||||
return []
|
||||
|
||||
seen_urls = set()
|
||||
|
||||
for link in links:
|
||||
try:
|
||||
href = link.get("href")
|
||||
if not href:
|
||||
continue
|
||||
|
||||
full_url = self.make_absolute_url(href)
|
||||
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
# Extract title
|
||||
title = link.get_text(strip=True)
|
||||
if not title:
|
||||
title = link.get("title", "")
|
||||
|
||||
# Try in parent
|
||||
if not title or len(title) < 3:
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
title_elem = parent.select_one("h2, h3, .product-title, [class*='title']")
|
||||
if title_elem:
|
||||
title = title_elem.get_text(strip=True)
|
||||
|
||||
if not title or len(title) < 3:
|
||||
continue
|
||||
|
||||
# Filter by query
|
||||
if not self.filter_by_query(title, query):
|
||||
continue
|
||||
|
||||
# Extract image
|
||||
image_url = None
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
img_selectors = [
|
||||
"img.product_img",
|
||||
"img.product-image",
|
||||
"img[class*='product']",
|
||||
"img",
|
||||
]
|
||||
for img_sel in img_selectors:
|
||||
img = parent.select_one(img_sel)
|
||||
if img:
|
||||
image_url = self._get_image_src(img)
|
||||
if image_url:
|
||||
image_url = self.make_absolute_url(image_url)
|
||||
break
|
||||
|
||||
# Extract price
|
||||
price = None
|
||||
if parent:
|
||||
price_selectors = [
|
||||
".price",
|
||||
"[class*='price'][class*='current']",
|
||||
"[data-testid='price']",
|
||||
"span[class*='prix']",
|
||||
]
|
||||
for price_sel in price_selectors:
|
||||
price_elem = parent.select_one(price_sel)
|
||||
if price_elem:
|
||||
price = self.parse_price_text(price_elem.get_text(strip=True))
|
||||
if price:
|
||||
break
|
||||
|
||||
products.append(ProductResult(
|
||||
title=title,
|
||||
url=full_url,
|
||||
source=self.site_name,
|
||||
price=price,
|
||||
currency="EUR",
|
||||
image_url=image_url,
|
||||
snippet=f"Product from {self.site_name}"
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error parsing Darty product: {e}")
|
||||
continue
|
||||
|
||||
self.logger.info(f"Extracted {len(products)} products from Darty")
|
||||
return products
|
||||
Reference in new issue
Block a user