mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-23 07:08:12 +02:00
feat: Rebuild search system with modular, site-specific parsers
Complete refactoring of the search module for robustness and reliability: **New Architecture:** - Modular parser system with specialized parsers for each major site - Factory pattern for automatic parser selection - Generic fallback parser for sites without specific implementation **Specialized Parsers:** - ✅ Amazon.fr - Robust parser with multiple fallback selectors - Handles sponsored products, Prime eligibility, ratings, reviews - Based on proven amazon_scraper_v2.py implementation - ✅ Cdiscount.com - Optimized for Cdiscount's HTML structure - ✅ Fnac.com - Handles Fnac's picture elements and article structure - ✅ Darty.com - Specialized for electronics retailer - ✅ Boulanger.com - Optimized for Boulanger's product cards **Key Features:** - Multiple fallback selectors for each element (title, price, image) - Proper price parsing for French format (1 234,56 €) - Rating and review count extraction - Stock status detection - Image URL extraction with multiple attribute fallbacks - Query filtering to ensure relevant results **Benefits:** - Much more robust than previous generic parsing - Easier to maintain (one parser per site) - Easily extensible (add new parser for new site) - Better error handling and logging - Higher success rate for product extraction **Updated Services:** - improved_search_service.py now uses ParserFactory - Legacy _parse_results marked as deprecated - Conversion layer between ProductResult and SearchResult This implements the Amazon France model across all major sites for consistent, reliable product search results.
This commit is contained in:
10 files changed
+1257
-5
No files matched your search
@@ -1,14 +1,15 @@
|
||||
"""
|
||||
Improved Search Service - Using Persistent Browser Connection
|
||||
Based on ScraperService pattern for better session management and reliability
|
||||
|
||||
UPDATED: Now uses modular, site-specific parsers for robust product extraction
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import AsyncGenerator
|
||||
from urllib.parse import quote_plus, urljoin
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from playwright.async_api import Browser, BrowserContext, Page, async_playwright
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -16,6 +17,7 @@ from sqlalchemy.orm import Session
|
||||
from app.core.search_config import SITE_CONFIGS, BROWSERLESS_URL
|
||||
from app.models import SearchSite
|
||||
from app.schemas import SearchProgress, SearchResultItem
|
||||
from app.services.parsers import ParserFactory, ProductResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -245,8 +247,12 @@ class ImprovedSearchService:
|
||||
logger.warning(f"⚠️ Page too small - possibly blocked")
|
||||
return []
|
||||
|
||||
# Parse results
|
||||
results = cls._parse_results(html_content, site_key, search_url, query)
|
||||
# Parse results using specialized parser
|
||||
parser = ParserFactory.get_parser(site_key)
|
||||
parsed_products = parser.parse_search_results(html_content, query, search_url)
|
||||
|
||||
# Convert ProductResult to SearchResult
|
||||
results = [cls._convert_to_search_result(p) for p in parsed_products]
|
||||
|
||||
# Scrape details for each result (in parallel)
|
||||
if results:
|
||||
@@ -271,9 +277,26 @@ class ImprovedSearchService:
|
||||
logger.info(f"✅ Successfully found {len(results)} products from {config['name']}")
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_search_result(product: ProductResult) -> SearchResult:
|
||||
"""Convert ProductResult to SearchResult"""
|
||||
return SearchResult(
|
||||
url=product.url,
|
||||
title=product.title,
|
||||
snippet=product.snippet,
|
||||
source=product.source,
|
||||
price=product.price,
|
||||
currency=product.currency,
|
||||
in_stock=product.in_stock,
|
||||
image_url=product.image_url,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_results(html: str, site_key: str, base_url: str, query: str) -> list[SearchResult]:
|
||||
"""Parse HTML content to extract search results"""
|
||||
"""
|
||||
DEPRECATED: Legacy parsing method - now using specialized parsers
|
||||
This method is kept for backward compatibility but should not be used
|
||||
"""
|
||||
config = SITE_CONFIGS[site_key]
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
Reference in new issue
Block a user