mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-24 15:48:11 +02:00
feat: Implement improved search service with centralized configuration and site-specific parsers for various retailers.
This commit is contained in:
Michael committed
1 parent
a8bffaec08
commit
c2f042beee
17 files changed
+829
-476
No files matched your search
@@ -1,131 +1,73 @@
|
||||
"""
|
||||
Darty Parser - Specialized parser for Darty.com
|
||||
"""
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from app.services.parsers.base_parser import BaseParser, ProductResult
|
||||
import logging
|
||||
|
||||
from .base_parser import BaseParser, ProductResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class DartyParser(BaseParser):
|
||||
"""
|
||||
Parser for Darty.com
|
||||
|
||||
Key selectors:
|
||||
- Product links: a[href*='/nav/achat/'][href*='.html']
|
||||
- Images: img.product_img
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
site_name="Darty",
|
||||
base_url="https://www.darty.com"
|
||||
)
|
||||
super().__init__("darty.com", "https://www.darty.com")
|
||||
|
||||
def parse_search_results(self, html: str, query: str, search_url: str) -> list[ProductResult]:
|
||||
"""Parse Darty search results"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
products = []
|
||||
|
||||
# Try multiple selectors
|
||||
selectors = [
|
||||
"a[href*='/nav/achat/'][href*='.html']",
|
||||
".product-card a",
|
||||
"[class*='product'] a[href*='/nav/achat/']",
|
||||
]
|
||||
|
||||
links = []
|
||||
for selector in selectors:
|
||||
links = soup.select(selector)
|
||||
if links:
|
||||
self.logger.info(f"Found {len(links)} products with selector: {selector}")
|
||||
break
|
||||
|
||||
if not links:
|
||||
self.logger.warning("No products found on Darty")
|
||||
return []
|
||||
|
||||
seen_urls = set()
|
||||
|
||||
for link in links:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
# Darty products
|
||||
# Config: a[href*='/nav/achat/'][href*='.html']
|
||||
|
||||
cards = soup.select(".product-card, .product_item, div[class*='product-card']")
|
||||
|
||||
for card in cards:
|
||||
try:
|
||||
href = link.get("href")
|
||||
# Link
|
||||
link_el = card.select_one("a[href*='/nav/achat/'], a.product-link")
|
||||
if not link_el:
|
||||
# Try finding any link if specific selector fails
|
||||
link_el = card.find("a")
|
||||
|
||||
if not link_el:
|
||||
continue
|
||||
|
||||
href = link_el.get('href')
|
||||
if not href:
|
||||
continue
|
||||
|
||||
full_url = self.make_absolute_url(href)
|
||||
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
# Extract title
|
||||
title = link.get_text(strip=True)
|
||||
|
||||
url = self.make_absolute_url(href)
|
||||
|
||||
# Title
|
||||
title = None
|
||||
title_el = card.select_one(".product_name, .product-title, h2, h3")
|
||||
if title_el:
|
||||
title = title_el.get_text(strip=True)
|
||||
else:
|
||||
title = link_el.get_text(strip=True)
|
||||
|
||||
if not title:
|
||||
title = link.get("title", "")
|
||||
|
||||
# Try in parent
|
||||
if not title or len(title) < 3:
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
title_elem = parent.select_one("h2, h3, .product-title, [class*='title']")
|
||||
if title_elem:
|
||||
title = title_elem.get_text(strip=True)
|
||||
|
||||
if not title or len(title) < 3:
|
||||
continue
|
||||
|
||||
# Filter by query
|
||||
if not self.filter_by_query(title, query):
|
||||
continue
|
||||
|
||||
# Extract image
|
||||
image_url = None
|
||||
parent = link.find_parent(["article", "li", "div"])
|
||||
if parent:
|
||||
img_selectors = [
|
||||
"img.product_img",
|
||||
"img.product-image",
|
||||
"img[class*='product']",
|
||||
"img",
|
||||
]
|
||||
for img_sel in img_selectors:
|
||||
img = parent.select_one(img_sel)
|
||||
if img:
|
||||
image_url = self._get_image_src(img)
|
||||
if image_url:
|
||||
image_url = self.make_absolute_url(image_url)
|
||||
break
|
||||
|
||||
# Extract price
|
||||
# Image
|
||||
img_url = self.extract_image_url(card)
|
||||
|
||||
# Price
|
||||
price = None
|
||||
if parent:
|
||||
price_selectors = [
|
||||
".price",
|
||||
"[class*='price'][class*='current']",
|
||||
"[data-testid='price']",
|
||||
"span[class*='prix']",
|
||||
]
|
||||
for price_sel in price_selectors:
|
||||
price_elem = parent.select_one(price_sel)
|
||||
if price_elem:
|
||||
price = self.parse_price_text(price_elem.get_text(strip=True))
|
||||
if price:
|
||||
break
|
||||
|
||||
products.append(ProductResult(
|
||||
price_el = card.select_one(".product_price, .price, [class*='price']")
|
||||
if price_el:
|
||||
price = self.parse_price_text(price_el.get_text())
|
||||
|
||||
results.append(ProductResult(
|
||||
title=title,
|
||||
url=full_url,
|
||||
source=self.site_name,
|
||||
url=url,
|
||||
source="Darty",
|
||||
price=price,
|
||||
currency="EUR",
|
||||
image_url=image_url,
|
||||
snippet=f"Product from {self.site_name}"
|
||||
in_stock=True,
|
||||
image_url=img_url,
|
||||
snippet=f"Product from Darty"
|
||||
))
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error parsing Darty product: {e}")
|
||||
logger.error(f"Error parsing Darty product: {e}")
|
||||
continue
|
||||
|
||||
self.logger.info(f"Extracted {len(products)} products from Darty")
|
||||
return products
|
||||
|
||||
logger.info(f"DartyParser found {len(results)} results")
|
||||
return results
|
||||
Reference in new issue
Block a user