mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-18 12:48:53 +02:00
feat: Implement improved search service with supporting models, schemas, and API routers, and refactor site verification script.
This commit is contained in:
Michael committed
1 parent
4720ed6b93
commit
9855040f3e
21 files changed
+57932
-419
No files matched your search
@@ -446,6 +446,15 @@ class BrowserlessService:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Generic wait selector
|
||||
if wait_selector:
|
||||
try:
|
||||
logger.info(f"⏳ Waiting for selector: {wait_selector}")
|
||||
await page.wait_for_selector(wait_selector, timeout=10000, state="attached")
|
||||
logger.info(f"✅ Wait selector found: {wait_selector}")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Wait selector {wait_selector} timed out or failed: {e}")
|
||||
|
||||
# Extract content
|
||||
if extract_text:
|
||||
|
||||
@@ -175,143 +175,6 @@ class ImprovedSearchService:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
await page.keyboard.press("Escape")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
async def search_site(cls, site_key: str, query: str) -> list[SearchResult]:
|
||||
"""
|
||||
Search a single site using persistent browser
|
||||
|
||||
Args:
|
||||
site_key: Site configuration key
|
||||
query: Search query
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects
|
||||
"""
|
||||
config = SITE_CONFIGS.get(site_key)
|
||||
if not config:
|
||||
logger.error(f"Unknown site: {site_key}")
|
||||
return []
|
||||
|
||||
search_url = config["search_url"].format(query=quote_plus(query))
|
||||
logger.info(f"🔍 Searching {config['name']} at {search_url}")
|
||||
|
||||
# Retry logic
|
||||
max_retries = 1
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
# Ensure browser is connected (check before each attempt)
|
||||
if not await cls._ensure_browser_connected():
|
||||
logger.error("Failed to establish browser connection")
|
||||
if attempt < max_retries:
|
||||
continue
|
||||
return []
|
||||
|
||||
context = await cls._create_context(cls._browser)
|
||||
try:
|
||||
page = await context.new_page()
|
||||
|
||||
try:
|
||||
# Navigate to search page
|
||||
logger.debug(f"Navigating to {search_url}")
|
||||
await page.goto(search_url, wait_until="domcontentloaded", timeout=30000)
|
||||
logger.debug("Page loaded (domcontentloaded)")
|
||||
|
||||
# Wait for network idle
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=10000)
|
||||
logger.debug("Network idle reached")
|
||||
except PlaywrightTimeoutError:
|
||||
logger.debug("Network idle timed out (non-critical)")
|
||||
|
||||
# Handle popups
|
||||
await cls._handle_popups(page)
|
||||
|
||||
# Wait for content to load
|
||||
wait_selector = config.get("wait_selector")
|
||||
if wait_selector:
|
||||
try:
|
||||
await page.wait_for_selector(wait_selector, timeout=5000)
|
||||
logger.debug(f"Wait selector found: {wait_selector}")
|
||||
except PlaywrightTimeoutError:
|
||||
logger.warning(f"Wait selector not found: {wait_selector}")
|
||||
|
||||
# Small delay for JS rendering
|
||||
await page.wait_for_timeout(2000)
|
||||
|
||||
# Get HTML content
|
||||
html_content = await page.content()
|
||||
logger.info(f"✅ Page content extracted ({len(html_content)} bytes)")
|
||||
|
||||
if len(html_content) < 5000:
|
||||
logger.warning(f"⚠️ Page too small - possibly blocked")
|
||||
return []
|
||||
|
||||
# Parse results using specialized parser
|
||||
parser = ParserFactory.get_parser(site_key)
|
||||
parsed_products = parser.parse_search_results(html_content, query, search_url)
|
||||
|
||||
# Convert ProductResult to SearchResult
|
||||
results = [cls._convert_to_search_result(p) for p in parsed_products]
|
||||
|
||||
# Scrape details for each result (in parallel)
|
||||
if results:
|
||||
logger.info(f"📦 Found {len(results)} initial results, enriching with details...")
|
||||
|
||||
# Enrich all results
|
||||
results_to_enrich = results
|
||||
logger.info(f"⚡ Enriching all {len(results_to_enrich)} products")
|
||||
|
||||
semaphore = asyncio.Semaphore(4) # Increased concurrency slightly
|
||||
|
||||
async def scrape_with_limit(res):
|
||||
async with semaphore:
|
||||
return await cls._scrape_item_details(res, context)
|
||||
|
||||
tasks = [scrape_with_limit(r) for r in results_to_enrich]
|
||||
enriched_results = await asyncio.gather(*tasks)
|
||||
|
||||
# Filter out failed enrichments if any (though scrape_item_details returns original on failure)
|
||||
results = [r for r in enriched_results if r]
|
||||
|
||||
# If we got here, success!
|
||||
logger.info(f"✅ Successfully found {len(results)} products from {config['name']}")
|
||||
return results
|
||||
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
finally:
|
||||
try:
|
||||
await context.close()
|
||||
except Exception as e:
|
||||
logger.debug(f"Error closing context (ignored): {e}")
|
||||
|
||||
except (PlaywrightTimeoutError, Exception) as e:
|
||||
# Check if it's a connection error or target closed
|
||||
error_msg = str(e).lower()
|
||||
is_connection_error = "target closed" in error_msg or "connection" in error_msg or "browser has been closed" in error_msg
|
||||
|
||||
if is_connection_error and attempt < max_retries:
|
||||
logger.warning(f"⚠️ Connection error during search for {site_key}: {e}. Retrying ({attempt+1}/{max_retries})...")
|
||||
# Force reconnect
|
||||
cls._browser = None
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
logger.error(f"❌ Error during search: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_search_result(product: ProductResult) -> SearchResult:
|
||||
"""Convert ProductResult to SearchResult"""
|
||||
return SearchResult(
|
||||
url=product.url,
|
||||
title=product.title,
|
||||
snippet=product.snippet,
|
||||
@@ -601,12 +464,6 @@ class ImprovedSearchService:
|
||||
return None # Unknown
|
||||
|
||||
|
||||
@classmethod
|
||||
async def search_site_generator(cls, site_key: str, query: str) -> AsyncGenerator[SearchResult, None]:
|
||||
"""Search a single site and yield results as they are scraped"""
|
||||
results = await cls.search_site(site_key, query)
|
||||
for result in results:
|
||||
yield result
|
||||
|
||||
@classmethod
|
||||
async def search_all(cls, query: str) -> list[SearchResult]:
|
||||
|
||||
+52
-116
@@ -78,7 +78,7 @@ class NewSearchService:
|
||||
logger.warning(f"No content returned for {site_key}")
|
||||
return []
|
||||
|
||||
return NewSearchService._parse_results(html_content, site_key, search_url)
|
||||
return NewSearchService._parse_results(html_content, site_key, search_url, query)
|
||||
|
||||
@staticmethod
|
||||
async def scrape_item(result: SearchResult) -> SearchResult:
|
||||
@@ -153,113 +153,73 @@ class NewSearchService:
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
async def search_site(site_key: str, query: str) -> list[SearchResult]:
|
||||
"""Search a single site"""
|
||||
config = SITE_CONFIGS.get(site_key)
|
||||
if not config:
|
||||
logger.error(f"Unknown site: {site_key}")
|
||||
return []
|
||||
|
||||
search_url = config["search_url"].format(query=quote_plus(query))
|
||||
logger.info(f"Searching {config['name']} at {search_url}")
|
||||
|
||||
# Use proxy if required by config
|
||||
use_proxy = config.get("requires_proxy", False)
|
||||
|
||||
html_content, _ = await browserless_service.get_page_content(
|
||||
search_url,
|
||||
use_proxy=use_proxy,
|
||||
wait_selector=config.get("wait_selector")
|
||||
)
|
||||
|
||||
if not html_content:
|
||||
logger.warning(f"No content returned for {site_key}")
|
||||
return []
|
||||
|
||||
# Phase 1: Parse results
|
||||
initial_results = NewSearchService._parse_results(html_content, site_key, search_url, query)
|
||||
|
||||
# Phase 2: Scrape details for each result (Parallel)
|
||||
# Limit concurrency to avoid overloading
|
||||
semaphore = asyncio.Semaphore(3)
|
||||
|
||||
async def scrape_with_limit(res):
|
||||
async with semaphore:
|
||||
return await NewSearchService.scrape_item(res)
|
||||
|
||||
tasks = [scrape_with_limit(r) for r in initial_results]
|
||||
enriched_results = await asyncio.gather(*tasks)
|
||||
|
||||
return enriched_results
|
||||
|
||||
@staticmethod
|
||||
def _parse_results(html: str, site_key: str, base_url: str, query: str) -> list[SearchResult]:
|
||||
def _parse_results(content: str, site: str, base_url: str, query: str) -> list[SearchResult]:
|
||||
"""Parse HTML content to extract search results"""
|
||||
config = SITE_CONFIGS[site_key]
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
config = SITE_CONFIGS[site]
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
|
||||
# Log content length and selector
|
||||
logger.debug(f"Parsing content for {site} (length: {len(content)}) with selector: {config['product_selector']}")
|
||||
|
||||
# Prepare query words for filtering
|
||||
# Split by whitespace, lowercase, remove special chars if needed, filter out short words
|
||||
query_words = [w.lower() for w in query.split() if len(w) > 2]
|
||||
|
||||
# Select product links
|
||||
links = soup.select(config["product_selector"])
|
||||
|
||||
# Deduplicate links
|
||||
seen_urls = set()
|
||||
|
||||
for link in links:
|
||||
# Limit removed as per user request
|
||||
# if len(results) >= 5:
|
||||
# break
|
||||
logger.debug(f"Found {len(links)} raw items for {site}")
|
||||
|
||||
href = link.get("href")
|
||||
if "amazon" in site:
|
||||
base_url = "https://www.amazon.fr"
|
||||
|
||||
seen_urls = set()
|
||||
query_words = query.lower().split() if query else []
|
||||
|
||||
for container in links:
|
||||
# Handle container-based selectors (where the selector is the card, not the link)
|
||||
link = None
|
||||
href = None
|
||||
title = None
|
||||
|
||||
# Special handling for sites where selector targets a container (Carrefour, Stokomani)
|
||||
# Try to find link within container
|
||||
if "product_link_selector" in config:
|
||||
link_el = container.select_one(config["product_link_selector"])
|
||||
if link_el:
|
||||
link = link_el
|
||||
href = link.get("href")
|
||||
|
||||
# Fallback: check if container itself is a link
|
||||
if not href:
|
||||
href = container.get("href")
|
||||
if href:
|
||||
link = container
|
||||
|
||||
# Special handling for sites where selector targets a container but no explicit link selector
|
||||
if not href and config.get("name") in ["Carrefour", "Stokomani"]:
|
||||
# Try to find the main product link inside the container
|
||||
# For Carrefour, it's usually .product-card-click-wrapper, but generic 'a' often works if it's the first one
|
||||
child_link = link.find("a", class_="product-card-click-wrapper") or link.find("a")
|
||||
child_link = container.find("a", class_="product-card-click-wrapper") or container.find("a")
|
||||
if child_link:
|
||||
href = child_link.get("href")
|
||||
# Update link to point to the anchor for title/image extraction
|
||||
link = child_link
|
||||
|
||||
if not href:
|
||||
# logger.debug(f"Skipping result: No href found for {config['name']}")
|
||||
continue
|
||||
|
||||
full_url = urljoin(base_url, href)
|
||||
|
||||
# Basic cleanup
|
||||
if full_url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(full_url)
|
||||
|
||||
# Extract title
|
||||
title = link.get_text(strip=True)
|
||||
title = None
|
||||
if "product_title_selector" in config:
|
||||
# Use container to find title
|
||||
title_el = container.select_one(config["product_title_selector"])
|
||||
if title_el:
|
||||
title = title_el.get_text(strip=True)
|
||||
|
||||
# If no text, check title attribute or nested image alt
|
||||
if not title:
|
||||
if link.get("title"):
|
||||
title = link.get("title")
|
||||
else:
|
||||
img = link.find("img")
|
||||
if img and img.get("alt"):
|
||||
title = img.get("alt")
|
||||
|
||||
# Special case for Stokomani or similar where link might be wrapping text but get_text failed or we selected a container
|
||||
if not title and config.get("name") == "Stokomani":
|
||||
# If we selected the container .product-card__title, the link is inside
|
||||
child_link = link.find("a")
|
||||
if child_link:
|
||||
title = child_link.get_text(strip=True)
|
||||
if not href: # Update href if we selected a container
|
||||
href = child_link.get("href")
|
||||
if href:
|
||||
full_url = urljoin(base_url, href)
|
||||
if not title and link:
|
||||
title = link.get_text(strip=True)
|
||||
|
||||
if not title or len(title) < 3:
|
||||
logger.debug(f"Skipping result: No title or too short ({title}) for {full_url}")
|
||||
continue
|
||||
|
||||
# STRICT FILTERING: Check if all query words are in the title
|
||||
@@ -272,7 +232,7 @@ class NewSearchService:
|
||||
break
|
||||
|
||||
if not all_words_found:
|
||||
# logger.debug(f"Skipping result '{title}' - does not contain all query words: {query_words}")
|
||||
logger.debug(f"Skipping result '{title}' - does not contain all query words: {query_words}")
|
||||
continue
|
||||
|
||||
# Extract Image URL (Enhanced with multi-selector support)
|
||||
@@ -284,17 +244,10 @@ class NewSearchService:
|
||||
img_el = None
|
||||
# Try each selector in order
|
||||
for selector in selectors:
|
||||
# Search in the link itself first
|
||||
img_el = link.select_one(selector)
|
||||
# Search in the container first
|
||||
img_el = container.select_one(selector)
|
||||
if img_el:
|
||||
break
|
||||
|
||||
# If not found in link, search in parent container
|
||||
container = link.find_parent("article") or link.find_parent("div", class_=lambda x: x and "product" in x)
|
||||
if container:
|
||||
img_el = container.select_one(selector)
|
||||
if img_el:
|
||||
break
|
||||
|
||||
if img_el:
|
||||
# Try multiple image attributes in order of priority
|
||||
@@ -311,9 +264,9 @@ class NewSearchService:
|
||||
srcset = img_el.get("srcset")
|
||||
image_url = srcset.split(",")[0].split()[0]
|
||||
|
||||
# Fallback: Find any img in the link
|
||||
# Fallback: Find any img in the container
|
||||
if not image_url:
|
||||
img = link.find("img")
|
||||
img = container.find("img")
|
||||
if img:
|
||||
image_url = (
|
||||
img.get("src") or
|
||||
@@ -328,35 +281,18 @@ class NewSearchService:
|
||||
srcset = img.get("srcset")
|
||||
image_url = srcset.split(",")[0].split()[0]
|
||||
|
||||
# Fallback: Search in parent container
|
||||
if not image_url:
|
||||
container = link.find_parent("article") or link.find_parent("div", class_=lambda x: x and "product" in x)
|
||||
if container:
|
||||
img = container.find("img")
|
||||
if img:
|
||||
image_url = (
|
||||
img.get("src") or
|
||||
img.get("data-src") or
|
||||
img.get("data-lazy-src") or
|
||||
img.get("data-original") or
|
||||
img.get("data-lazy")
|
||||
)
|
||||
if not image_url and img.get("srcset"):
|
||||
srcset = img.get("srcset")
|
||||
image_url = srcset.split(",")[0].split()[0]
|
||||
|
||||
# Make absolute URL
|
||||
if image_url:
|
||||
# Clean up data URIs or invalid URLs
|
||||
if image_url.startswith("data:"):
|
||||
logger.debug(f"Skipping data URI for: {title[:30]}")
|
||||
# logger.debug(f"Skipping data URI for: {title[:30]}")
|
||||
image_url = None
|
||||
elif not image_url.startswith("http"):
|
||||
image_url = urljoin(base_url, image_url)
|
||||
|
||||
# Log if image not found
|
||||
if not image_url:
|
||||
logger.warning(f"No image found for: {title[:50]} | {site_key}")
|
||||
logger.warning(f"No image found for: {title[:50]} | {site}")
|
||||
|
||||
|
||||
# Create result
|
||||
@@ -368,7 +304,7 @@ class NewSearchService:
|
||||
image_url=image_url
|
||||
))
|
||||
|
||||
logger.info(f"Found {len(results)} results for {site_key}")
|
||||
logger.info(f"Found {len(results)} results for {site}")
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in new issue
Block a user