From 03793cea0a9c0d3e86279a59fb138eff8cfb5e1c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:23:24 +0000 Subject: [PATCH] fix: Add extract_text parameter to fix broken product search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** Previous commit changed get_page_content() to always return visible text (inner_text) instead of HTML source. This broke product search which uses BeautifulSoup to parse HTML with CSS selectors. Search was returning 0 results for all sites. **Root Cause:** - Search service uses: `soup.select(config["product_selector"])` on HTML - But was receiving plain text instead of HTML structure - BeautifulSoup couldn't find any product links → 0 results **Solution: Add extract_text parameter** Modified `get_page_content()` signature: ```python async def get_page_content( url: str, use_proxy: bool = False, wait_selector: str = None, extract_text: bool = False # NEW parameter ) -> tuple[str, str]: ``` **Behavior:** - `extract_text=False` (DEFAULT): Returns HTML source via `page.content()` → Used by search service for BeautifulSoup parsing - `extract_text=True`: Returns visible text via `page.inner_text('body')` → Used by AI monitoring for price extraction → Includes Amazon price prepending ("PRIX DÉTECTÉ: ...") **Changes:** 1. **browserless_service.py** (Lines 197-329) - Added `extract_text` parameter with default `False` - Conditional logic: if extract_text, use inner_text + Amazon extraction - Else: use page.content() (HTML source) - Preserves backward compatibility (default = HTML) 2. **scheduler_service.py** (Line 113) - Added `extract_text=True` for AI monitoring - Ensures AI gets visible text with Amazon prices 3. **search_service.py** (Line 104) - Added `extract_text=True` for scrape_item() AI analysis - Search itself uses default (HTML) for BeautifulSoup parsing **Impact:** ✅ Search now works again (gets HTML for BeautifulSoup) ✅ AI monitoring gets visible text (better extraction) ✅ Amazon price extraction only runs when extract_text=True ✅ Backward compatible (default behavior = HTML) Fixes issue where all searches returned: "Found 0 results for [site]" --- app/services/browserless_service.py | 55 +++++++++++++++++++---------- app/services/scheduler_service.py | 4 ++- app/services/search_service.py | 10 +++--- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 6abd850..f894119 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -194,10 +194,24 @@ class BrowserlessService: logger.warning("Could not extract Amazon price with any selector") return "" - async def get_page_content(self, url: str, use_proxy: bool = False, wait_selector: str = None) -> tuple[str, str]: + async def get_page_content( + self, + url: str, + use_proxy: bool = False, + wait_selector: str = None, + extract_text: bool = False + ) -> tuple[str, str]: """ Fetch page content with full stealth lifecycle. - Returns (html_content, screenshot_path) + + Args: + url: URL to fetch + use_proxy: Whether to use proxy rotation + wait_selector: CSS selector to wait for before capturing + extract_text: If True, returns visible text; if False, returns HTML source + + Returns: + (content, screenshot_path) where content is either HTML or visible text """ context = await self.get_context(use_proxy=use_proxy) page = await context.new_page() @@ -291,23 +305,28 @@ class BrowserlessService: except Exception: pass - # Extract visible text instead of raw HTML - # This gets the text as rendered by JavaScript, not the HTML source - try: - content = await page.inner_text('body') - logger.info(f"Extracted {len(content)} chars of visible text from page") - except Exception as e: - # Fallback to HTML content if inner_text fails - logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") - content = await page.content() + # Extract content based on extract_text parameter + if extract_text: + # Extract visible text for AI analysis (monitoring use case) + try: + content = await page.inner_text('body') + logger.info(f"Extracted {len(content)} chars of visible text from page") + except Exception as e: + # Fallback to HTML content if inner_text fails + logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") + content = await page.content() - # For Amazon, extract price directly and prepend to content - # This helps AI by providing explicit price information - if "amazon" in url.lower() and "/dp/" in url: - amazon_price_text = await self._extract_amazon_price(page) - if amazon_price_text: - content = f"PRIX DÉTECTÉ: {amazon_price_text}\n\n{content}" - logger.info(f"Prepended Amazon price to content: {amazon_price_text}") + # For Amazon, extract price directly and prepend to content + # This helps AI by providing explicit price information + if "amazon" in url.lower() and "/dp/" in url: + amazon_price_text = await self._extract_amazon_price(page) + if amazon_price_text: + content = f"PRIX DÉTECTÉ: {amazon_price_text}\n\n{content}" + logger.info(f"Prepended Amazon price to content: {amazon_price_text}") + else: + # Extract raw HTML for parsing (search use case) + content = await page.content() + logger.debug(f"Extracted {len(content)} chars of HTML from page") # Take screenshot if needed for AI analysis diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 04eef86..de8f1fd 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -105,10 +105,12 @@ async def process_item_check(item_id: int): logger.info(f"Checking item: {item_data['name']} ({item_data['url']})") # Use new browserless service # Note: smart_scroll is handled internally by browserless_service + # extract_text=True to get visible text for AI analysis (not HTML) page_text, screenshot_path = await browserless_service.get_page_content( item_data["url"], use_proxy="amazon" in item_data["url"], # Simple heuristic for now - wait_selector=item_data["selector"] + wait_selector=item_data["selector"], + extract_text=True # Get visible text for AI, not raw HTML ) # Determine availability based on content presence diff --git a/app/services/search_service.py b/app/services/search_service.py index 779cce4..21ce2d1 100644 --- a/app/services/search_service.py +++ b/app/services/search_service.py @@ -96,18 +96,20 @@ class NewSearchService: use_proxy = config.get("requires_proxy", False) if config else False # Use browserless to get content and screenshot - html, screenshot_path = await browserless_service.get_page_content( + # extract_text=True to get visible text for AI analysis + page_text, screenshot_path = await browserless_service.get_page_content( result.url, use_proxy=use_proxy, - wait_selector=None + wait_selector=None, + extract_text=True # Get visible text for AI price extraction ) - + if not screenshot_path: return result # Use AI to analyze from app.services.ai_service import AIService - ai_result = await AIService.analyze_image(screenshot_path, page_text=html) + ai_result = await AIService.analyze_image(screenshot_path, page_text=page_text) if ai_result: extraction, _ = ai_result