From 12b281f609996045b6c6357af742148505d953df Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 19:45:00 +0000 Subject: [PATCH] feat: Extract visible text and optimize Amazon price detection Critical improvements to AI price extraction for better accuracy: **1. Extract Visible Text Instead of HTML** (browserless_service.py) - Changed from `page.content()` (raw HTML) to `page.inner_text('body')` - Now captures text as rendered by JavaScript, not HTML source - Amazon and other sites inject prices via JavaScript - HTML source doesn't contain them - Includes proper fallback to HTML if inner_text fails **2. Amazon-Specific Price Detection** (browserless_service.py) - Wait for Amazon price elements to load before screenshot: - `.a-price .a-offscreen` (main price) - `#corePriceDisplay_desktop_feature_div` (price section) - `#corePrice_desktop` (alternative container) - `.a-price-whole` (price number) - Prevents screenshots before dynamic prices are rendered **3. Focused Amazon Screenshots** (browserless_service.py) - Problem: Full-page Amazon screenshots are huge (multiple screens tall) - When resized to 2048px, price text becomes tiny and unreadable - Solution: Take focused screenshot of product container only: - Try `#dp-container`, `#ppd`, or `#centerCol` (product areas) - Falls back to viewport screenshot (better than full-page) - Other sites still use full-page screenshots **4. Enhanced Debug Logging** (ai_service.py) - Log cleaned text preview (first 500 chars) to verify content - Log prompt preview to debug AI instructions - Warning when no page_text provided These changes address the root causes of AI returning: - price: null, confidence: 0.0-0.1 (too low to update DB) Expected improvement: - Visible text contains actual rendered prices - Smaller, focused screenshots = better AI recognition - Confidence scores should reach > 0.5 threshold --- app/services/ai_service.py | 6 +++ app/services/browserless_service.py | 67 +++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/app/services/ai_service.py b/app/services/ai_service.py index b2c9f1b..60096ae 100644 --- a/app/services/ai_service.py +++ b/app/services/ai_service.py @@ -343,8 +343,14 @@ class AIService: if len(cleaned_text) > MAX_TEXT_LENGTH: cleaned_text = cleaned_text[:MAX_TEXT_LENGTH] + "...(truncated)" logger.info(f"Added text context (original: {len(page_text)}, cleaned: {len(cleaned_text)})") + # Debug: Log first 500 chars of cleaned text to see what AI receives + logger.debug(f"Cleaned text preview: {cleaned_text[:500]!r}") + else: + logger.warning("No page_text provided - AI will only use screenshot") prompt = get_extraction_prompt(cleaned_text if cleaned_text else None) + # Debug: Log prompt preview + logger.debug(f"Prompt preview (first 300 chars): {prompt[:300]!r}") # Call LLM response_text = await cls.call_llm(prompt, data_url, config) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index cb75215..2348e05 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -215,13 +215,39 @@ class BrowserlessService: except Exception: logger.warning(f"Wait selector {wait_selector} timed out") + # For Amazon product pages, wait for price to load + if "amazon" in url.lower() and "/dp/" in url: + amazon_price_selectors = [ + ".a-price .a-offscreen", # Main price element + "#corePriceDisplay_desktop_feature_div", # Price section + "#corePrice_desktop", # Alternative price container + ".a-price-whole", # Price number + ] + for selector in amazon_price_selectors: + try: + await page.wait_for_selector(selector, timeout=5000, state="visible") + logger.info(f"Amazon price element found: {selector}") + break + except Exception: + continue + else: + logger.warning("No Amazon price selector found, proceeding anyway") + # Wait for network idle to ensure dynamic content loads try: await page.wait_for_load_state("networkidle", timeout=5000) except Exception: pass - content = await page.content() + # Extract visible text instead of raw HTML + # This gets the text as rendered by JavaScript, not the HTML source + try: + content = await page.inner_text('body') + logger.info(f"Extracted {len(content)} chars of visible text from page") + except Exception as e: + # Fallback to HTML content if inner_text fails + logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") + content = await page.content() # Take screenshot if needed for AI analysis screenshot_path = "" @@ -239,9 +265,42 @@ class BrowserlessService: filename = f"{safe_name}_{timestamp}.jpg" screenshot_path = f"{screenshots_dir}/{filename}" - # Use full_page=True to capture entire page, not just viewport - await page.screenshot(path=screenshot_path, full_page=True, quality=80, type="jpeg") - logger.info(f"Screenshot saved to {screenshot_path}") + # For Amazon, take focused screenshot of product info area + # Full-page screenshots of Amazon are too large and price becomes tiny when resized + is_amazon = "amazon" in url.lower() and "/dp/" in url + + if is_amazon: + # Try to screenshot just the main product area containing price + try: + # Look for main product container + product_selectors = [ + "#dp-container", # Main product container + "#ppd", # Product page data + "#centerCol", # Center column with price + ] + screenshot_taken = False + for selector in product_selectors: + try: + element = page.locator(selector).first + if await element.is_visible(): + await element.screenshot(path=screenshot_path, quality=85, type="jpeg") + logger.info(f"Amazon focused screenshot saved using {selector}") + screenshot_taken = True + break + except Exception: + continue + + if not screenshot_taken: + # Fallback to viewport screenshot (better than full-page for Amazon) + await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") + logger.info("Amazon viewport screenshot saved (fallback)") + except Exception as e: + logger.warning(f"Amazon focused screenshot failed: {e}, using viewport") + await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") + else: + # For non-Amazon sites, use full_page to capture entire page + await page.screenshot(path=screenshot_path, full_page=True, quality=80, type="jpeg") + logger.info(f"Full-page screenshot saved to {screenshot_path}") except Exception as e: logger.warning(f"Failed to take screenshot: {e}")