diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 6abd850..f894119 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -194,10 +194,24 @@ class BrowserlessService: logger.warning("Could not extract Amazon price with any selector") return "" - async def get_page_content(self, url: str, use_proxy: bool = False, wait_selector: str = None) -> tuple[str, str]: + async def get_page_content( + self, + url: str, + use_proxy: bool = False, + wait_selector: str = None, + extract_text: bool = False + ) -> tuple[str, str]: """ Fetch page content with full stealth lifecycle. - Returns (html_content, screenshot_path) + + Args: + url: URL to fetch + use_proxy: Whether to use proxy rotation + wait_selector: CSS selector to wait for before capturing + extract_text: If True, returns visible text; if False, returns HTML source + + Returns: + (content, screenshot_path) where content is either HTML or visible text """ context = await self.get_context(use_proxy=use_proxy) page = await context.new_page() @@ -291,23 +305,28 @@ class BrowserlessService: except Exception: pass - # Extract visible text instead of raw HTML - # This gets the text as rendered by JavaScript, not the HTML source - try: - content = await page.inner_text('body') - logger.info(f"Extracted {len(content)} chars of visible text from page") - except Exception as e: - # Fallback to HTML content if inner_text fails - logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") - content = await page.content() + # Extract content based on extract_text parameter + if extract_text: + # Extract visible text for AI analysis (monitoring use case) + try: + content = await page.inner_text('body') + logger.info(f"Extracted {len(content)} chars of visible text from page") + except Exception as e: + # Fallback to HTML content if inner_text fails + logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") + content = await page.content() - # For Amazon, extract price directly and prepend to content - # This helps AI by providing explicit price information - if "amazon" in url.lower() and "/dp/" in url: - amazon_price_text = await self._extract_amazon_price(page) - if amazon_price_text: - content = f"PRIX DÉTECTÉ: {amazon_price_text}\n\n{content}" - logger.info(f"Prepended Amazon price to content: {amazon_price_text}") + # For Amazon, extract price directly and prepend to content + # This helps AI by providing explicit price information + if "amazon" in url.lower() and "/dp/" in url: + amazon_price_text = await self._extract_amazon_price(page) + if amazon_price_text: + content = f"PRIX DÉTECTÉ: {amazon_price_text}\n\n{content}" + logger.info(f"Prepended Amazon price to content: {amazon_price_text}") + else: + # Extract raw HTML for parsing (search use case) + content = await page.content() + logger.debug(f"Extracted {len(content)} chars of HTML from page") # Take screenshot if needed for AI analysis diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 04eef86..de8f1fd 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -105,10 +105,12 @@ async def process_item_check(item_id: int): logger.info(f"Checking item: {item_data['name']} ({item_data['url']})") # Use new browserless service # Note: smart_scroll is handled internally by browserless_service + # extract_text=True to get visible text for AI analysis (not HTML) page_text, screenshot_path = await browserless_service.get_page_content( item_data["url"], use_proxy="amazon" in item_data["url"], # Simple heuristic for now - wait_selector=item_data["selector"] + wait_selector=item_data["selector"], + extract_text=True # Get visible text for AI, not raw HTML ) # Determine availability based on content presence diff --git a/app/services/search_service.py b/app/services/search_service.py index 779cce4..21ce2d1 100644 --- a/app/services/search_service.py +++ b/app/services/search_service.py @@ -96,18 +96,20 @@ class NewSearchService: use_proxy = config.get("requires_proxy", False) if config else False # Use browserless to get content and screenshot - html, screenshot_path = await browserless_service.get_page_content( + # extract_text=True to get visible text for AI analysis + page_text, screenshot_path = await browserless_service.get_page_content( result.url, use_proxy=use_proxy, - wait_selector=None + wait_selector=None, + extract_text=True # Get visible text for AI price extraction ) - + if not screenshot_path: return result # Use AI to analyze from app.services.ai_service import AIService - ai_result = await AIService.analyze_image(screenshot_path, page_text=html) + ai_result = await AIService.analyze_image(screenshot_path, page_text=page_text) if ai_result: extraction, _ = ai_result