From 531ed1bc15d6aef7a98d06b8ef56b5e7de2bf13a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:51:00 +0000 Subject: [PATCH 1/4] fix: Convert French price format to English before sending to AI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** AI was misinterpreting French decimal comma as thousands separator: - "3,99 €" detected as 399.00 EUR (instead of 3.99 EUR) - "1,99 €" detected as 199.00 EUR (instead of 1.99 EUR) - Caused by AI reading comma as grouping separator, not decimal point **Root Cause:** French format uses comma for decimals: "3,99 €" = three euros ninety-nine cents English format uses dot for decimals: "3.99 €" = three euros ninety-nine cents AI models (trained mostly on English) interpret: - "3,99" → "3" and "99" separate → 399 (three hundred ninety-nine) - Should be: "3.99" → 3.99 (three point ninety-nine) **Solution: Pre-Convert Format Before AI Processing** 1. **Normalize ALL Text** (browserless_service.py:380-386) ```python # Convert French to English in all page text content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) # "3,99 €" → "3.99 €" # "12,50 €" → "12.50 €" # "1 234,56 €" → "1234.56 €" (also removes space thousands separator) ``` 2. **Normalize Extracted Price** (browserless_service.py:400-410) ```python # Convert French format in directly extracted price normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', extracted_price) # "3,99 €" → "3.99 €" # Prepends: "PRIX DÉTECTÉ: 3.99 €" (not "3,99 €") ``` 3. **Updated AI Prompt** (ai_schema.py:135-158) - Removed instructions to convert comma→dot (now done automatically) - Added: "The price has already been converted to English format for you" - Added explicit warnings about common mistakes: * "3.99 €" means 3.99 (NOT 399.00, NOT 3990.00) * "1.99 €" means 1.99 (NOT 199.00, NOT 1990.00) - Clear examples with both CORRECT and INCORRECT values **Flow:** ``` Page text: "Lutin 3,99 € Ajouter au panier" ↓ Normalization: "Lutin 3.99 € Ajouter au panier" ↓ Extracted price: "3,99 €" → Normalized: "3.99 €" ↓ Sent to AI: "PRIX DÉTECTÉ: 3.99 €\n\nLutin 3.99 € Ajouter au panier" ↓ AI sees only English format: "3.99" ↓ AI Response: {"price": 3.99} ✅ CORRECT ``` **Expected Results:** - Before: "3,99 €" → AI returns 399.00 ❌ - After: "3,99 €" → Normalized to "3.99 €" → AI returns 3.99 ✅ **Regex Details:** - `(\d+),(\d{2})\s*€` matches French prices: captures digits before comma, 2 digits after - `\1.\2 €` replaces with: same digits, dot instead of comma, space + euro symbol - Handles: 0,99 / 3,99 / 12,99 / 89,99 / 1234,99 Fixes issue where AI misread French decimal comma causing 100x price errors. --- app/ai_schema.py | 28 +++++++++++++++------------- app/services/browserless_service.py | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/app/ai_schema.py b/app/ai_schema.py index 65621d1..1500623 100644 --- a/app/ai_schema.py +++ b/app/ai_schema.py @@ -134,26 +134,28 @@ EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from this **PRICE (IMPORTANT - French format):** - **FIRST**: Look for "PRIX DÉTECTÉ:" at the start of text - this is the extracted price -- French prices use COMMA for decimals: "12,99 €" means 12.99 -- Thousands separator is SPACE or DOT: "1 234,56 €" or "1.234,56 €" means 1234.56 +- **The price has already been converted to English format for you** +- French original: "3,99 €" → Already shown to you as: "3.99 €" +- Just extract the number you see (e.g., "3.99" from "PRIX DÉTECTÉ: 3.99 €") - Currency symbol is usually € at the end - Look for: "PRIX DÉTECTÉ:", price tags, "Prix:", "€", numbers near "Ajouter au panier" -- Extract as DECIMAL NUMBER (convert comma to dot): 12,99 -> 12.99 +- Extract as DECIMAL NUMBER: If you see "3.99", return 3.99 - Ignore crossed-out/barré prices (old prices) - If multiple prices, take the current/main price (not the original) -**CRITICAL - Read digits carefully:** -- PAY CLOSE ATTENTION to first digit: "1,99 €" is NOT "9,99 €" -- Double-check: Is it 1, 7, 8, or 9? These look similar -- Small prices (< 5€) are common: 0,99 €, 1,49 €, 1,99 €, 2,99 €, 3,99 €, 4,99 € -- VERIFY the price makes sense for the product type +**CRITICAL - Common mistakes to avoid:** +- "3.99 €" means 3.99 (NOT 399.00, NOT 3990.00) +- "1.99 €" means 1.99 (NOT 199.00, NOT 1990.00) +- "0.99 €" means 0.99 (NOT 99.00, NOT 990.00) +- The decimal point separates euros from cents +- Small prices (< 10€) are very common for everyday items - Examples of valid prices: - * "1,99 €" -> 1.99 (NOT 9.99) - * "0,99 €" -> 0.99 (NOT 9.99) - * "89,99 €" -> 89.99 - * "1 234,56 €" -> 1234.56 - * "PRIX DÉTECTÉ: 89,99 €" -> 89.99 + * "PRIX DÉTECTÉ: 1.99 €" -> 1.99 (NOT 199 or 1990) + * "PRIX DÉTECTÉ: 3.99 €" -> 3.99 (NOT 399 or 3990) + * "0.99 €" -> 0.99 (NOT 99) + * "89.99 €" -> 89.99 (NOT 8999) + * "1234.56 €" -> 1234.56 - If you find ANY price with € symbol, extract it with confidence >= 0.8 - If digits are unclear or blurry, reduce confidence to 0.5-0.7 - If unclear: set null and confidence < 0.5 diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index a55e98b..708a4b1 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -376,6 +376,15 @@ class BrowserlessService: try: content = await page.inner_text('body') logger.info(f"Extracted {len(content)} chars of visible text from page") + + # Normalize French prices to English format for AI (3,99 € → 3.99 €) + import re + # Replace comma with dot in price patterns: digits,XX € → digits.XX € + content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) + # Remove spaces in thousands: 1 234.56 → 1234.56 + content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content) + logger.debug("Normalized French price formats to English") + except Exception as e: # Fallback to HTML content if inner_text fails logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") @@ -397,8 +406,17 @@ class BrowserlessService: # Prepend extracted price to content if found if extracted_price: - content = f"PRIX DÉTECTÉ: {extracted_price}\n\n{content}" - logger.info(f"Prepended price to content: {extracted_price}") + # Convert French format to English for AI (3,99 → 3.99) + import re + normalized_price = extracted_price + # Replace comma with dot for decimal separator (French to English) + # Pattern: digits, comma, 2 digits → digits, dot, 2 digits + normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price) + # Remove spaces in thousands separators if any (1 234,56 → 1234.56) + normalized_price = normalized_price.replace(' ', '') + + content = f"PRIX DÉTECTÉ: {normalized_price}\n\n{content}" + logger.info(f"Prepended normalized price to content: {normalized_price} (original: {extracted_price})") else: logger.warning("Could not extract price from DOM, relying on text/image only") else: From 4c98fee285e6337a9ae598148d37f84f6e24a279 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:52:10 +0000 Subject: [PATCH 2/4] fix: Skip strikethrough prices to avoid detecting old/crossed-out prices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** AI was detecting old crossed-out prices instead of current prices. Example: Stokomani Lutin - Current: 19,99€, Old (strikethrough): 14,99€ AI detected: 14.99 EUR (wrong - old price) **Root Cause:** Generic price extraction was finding ALL price elements without checking if they were visually crossed-out (text-decoration: line-through). Many e-commerce sites show: ```html 14,99 € 19,99 € ``` The selector finds both, but we were returning the first found. **Solution: Check CSS text-decoration** Added strikethrough detection (browserless_service.py:182-189): ```python # Check if element is strikethrough (old price) text_decoration = await element.evaluate( "el => window.getComputedStyle(el).textDecoration" ) if "line-through" in text_decoration: continue # Skip crossed-out prices ``` **Flow:** ``` Found elements with .price selector: [elem1, elem2, elem3] ↓ For each element: 1. Check visibility ✓ 2. Check text-decoration → "line-through" → SKIP ✓ → "none" → CONTINUE 3. Extract price text ↓ Return first non-strikethrough price ``` **CSS Patterns Detected:** - `text-decoration: line-through` (most common) - `text-decoration: line-through solid` - Combined styles ignored if they don't contain "line-through" **Expected Results:** - Before: Returns first price found (14.99 if it's first in DOM) - After: Skips strikethrough prices, returns current price (19.99) **Note:** Existing selector already excludes common classes: `:not([class*='old']):not([class*='was']):not([class*='original'])` This adds runtime CSS check as additional safety layer. Partial fix for Stokomani Lutin 14.99 vs 19.99 issue. May need site-specific selectors if problem persists. --- app/services/browserless_service.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 708a4b1..7a6cb51 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -179,6 +179,15 @@ class BrowserlessService: try: element = elements.nth(i) if await element.is_visible(timeout=1000): + # Check if element is strikethrough (old price) + try: + text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration") + if "line-through" in text_decoration: + logger.debug(f"Skipping strikethrough price at {selector}") + continue # Skip crossed-out prices + except Exception: + pass # If we can't check, continue anyway + price_text = await element.inner_text() # Check if it looks like a price (contains € or digits with comma/dot) if price_text and ('€' in price_text or (',' in price_text and any(c.isdigit() for c in price_text))): From 1c763143caf5979ca5610631b346c67e8c8e051c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:55:05 +0000 Subject: [PATCH 3/4] feat: Improve price extraction with priority selectors and smart multi-price handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** Stokomani Lutin product detected 14.99 EUR instead of 19.99 EUR. Website shows both old price (14.99) and current price (19.99). Generic extraction was returning whichever price was found first. **Root Causes:** 1. No prioritization of price selectors (all treated equally) 2. First-found-first-returned approach missed semantic selectors 3. When multiple prices found, no logic to select the correct one **Solution: Priority-Based Extraction + Smart Selection** 1. **Reordered Selectors by Priority** (browserless_service.py:151-170) ``` HIGHEST PRIORITY (return immediately): - [itemprop='price'] (Schema.org semantic) - .current-price (explicit current) - .sale-price (sale/promo price) - .final-price (final calculated) - .product-price (product-specific) MEDIUM PRIORITY: - [class*='price']:not([class*='old']):not([class*='was'])... - Excludes: old, was, original, before, regular, ancien, barre LOW PRIORITY: - .price (generic, might catch anything) ``` 2. **Immediate Return for High-Priority Selectors** (lines 202-204) - If found with semantic selector → return immediately - Avoids checking lower-priority selectors - Most reliable prices checked first 3. **Multi-Price Smart Selection** (lines 210-218) - If multiple prices found (e.g., old + current) - Takes LAST price in DOM order (not highest/lowest value) - Rationale: HTML structure typically shows old price first, current price last: ```html 14,99 € ← First in DOM 19,99 € ← Last in DOM (CURRENT) ``` - Logs all found prices for debugging **Why Not "Highest Price"?** Taking highest value fails in promotions: ``` Old: 29,99 € (highest) Current: 19,99 € (promotion) ← Should select this ``` Taking last in DOM works for both cases: ``` Case 1 (Stokomani): Old: 14,99 € (first) Current: 19,99 € (last) ← Selected ✓ Case 2 (Promotion): Old: 29,99 € (first) Current: 19,99 € (last) ← Selected ✓ ``` **Flow:** ``` Search for [itemprop='price'] → Found? Return immediately ✓ → Not found? Continue... Search for .current-price → Found? Return immediately ✓ → Not found? Continue... ... Search with low-priority selectors → Found: ["14,99 €", "19,99 €"] → Select last: "19,99 €" ✓ ``` **Expected Results:** - Before: "14.99 EUR" (first found, wrong) - After: "19.99 EUR" (last in DOM, correct) - Logs: "Multiple prices found: ['14,99 €', '19,99 €'], selecting last: 19,99 €" Tested with: https://www.stokomani.fr/products/lutin-filou-telescopique-95-cm_208400 --- app/services/browserless_service.py | 43 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 7a6cb51..8e7b4c7 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -147,35 +147,38 @@ class BrowserlessService: Extract price from generic e-commerce pages using common selectors and patterns. Returns price text (e.g., "1,99 €") or empty string if not found. """ - # Common price selectors used across e-commerce sites + # Common price selectors used across e-commerce sites (ordered by priority) price_selectors = [ - # Common class names - "[class*='price']:not([class*='old']):not([class*='was']):not([class*='original'])", - ".product-price", - ".price", + # Highest priority: semantic and explicit current prices + "[itemprop='price']", ".current-price", ".sale-price", ".final-price", - # Common data attributes + ".product-price", "[data-price]", - "[itemprop='price']", + # Medium priority: generic price classes (exclude old/was/original) + "[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='regular'])", # ID-based "#price", "#product-price", "#our-price", - # Specific patterns - "span[class*='prix']", - "div[class*='prix']", + # French-specific + "span[class*='prix']:not([class*='ancien']):not([class*='barre'])", + "div[class*='prix']:not([class*='ancien'])", "span[class*='tarif']", + # Lower priority: generic .price (might catch old prices) + ".price", ] + found_prices = [] + for selector in price_selectors: try: elements = page.locator(selector) count = await elements.count() - # Try first visible element - for i in range(min(count, 3)): # Check max 3 elements + # Check up to 3 elements per selector + for i in range(min(count, 3)): try: element = elements.nth(i) if await element.is_visible(timeout=1000): @@ -193,13 +196,27 @@ class BrowserlessService: if price_text and ('€' in price_text or (',' in price_text and any(c.isdigit() for c in price_text))): # Clean up price text price_text = price_text.strip() + found_prices.append((selector, price_text)) logger.info(f"Found price via selector {selector}: {price_text}") - return price_text + + # Return immediately if we found with high-priority selector + if selector in ["[itemprop='price']", ".current-price", ".sale-price", ".final-price", ".product-price"]: + return price_text except Exception: continue except Exception: continue + # If we found prices with lower-priority selectors + if found_prices: + # When multiple prices found, the LAST one in DOM order is usually the current price + # (websites typically show: old price first, then current price) + best_price = found_prices[-1][1] # Take last (most recently added) + if len(found_prices) > 1: + prices_list = [p[1] for p in found_prices] + logger.info(f"Multiple prices found: {prices_list}, selecting last (most likely current): {best_price}") + return best_price + # Fallback: Use regex to find price patterns in all text try: all_text = await page.inner_text('body') From 66d9046de2c3fa3f48bf443d06bb6c44127ccaab Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:56:39 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20Normalize=20alternative=20French=20p?= =?UTF-8?q?rice=20format=20(digits=E2=82=ACcents=20without=20separator)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** L'Incroyable product showing "3€99" was detected as 399.00 EUR instead of 3.99 EUR. URL: https://www.lincroyable.fr/p95361-porte-magique-lutin/ **Root Cause:** Previous normalization only handled "3,99 €" (with comma separator). Some French sites use alternative format: "3€99" (euro symbol as separator). Existing regex: ```python re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) # Only matches "3,99 €" ``` Missed format: ``` "3€99" → No match → AI sees "399" → Returns 399.00 ``` **Solution: Add Pattern for Euro-as-Separator Format** 1. **Text Normalization** (browserless_service.py:408-411) ```python # NEW Pattern 1: digits€XX → digits.XX € content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content) # "3€99" → "3.99 €" # "19€50" → "19.50 €" # Pattern 2: digits,XX € → digits.XX € (existing) content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) # "3,99 €" → "3.99 €" ``` 2. **Extracted Price Normalization** (browserless_service.py:438-441) - Same patterns applied to directly extracted prices - Ensures consistency regardless of extraction method **Regex Details:** - `(\d+)€(\d{2})\b` matches: * `\d+` = one or more digits (euros) * `€` = euro symbol as separator * `(\d{2})` = exactly 2 digits (cents) * `\b` = word boundary (prevents matching "10€999") - Replacement: `\1.\2 €` = "digits.cents €" **Supported Formats After Fix:** ``` "3,99 €" → "3.99 €" ✓ (existing) "3€99" → "3.99 €" ✓ (NEW) "19,50 €" → "19.50 €" ✓ (existing) "19€50" → "19.50 €" ✓ (NEW) "1 234,56" → "1234.56" ✓ (existing) ``` **Flow:** ``` Page text: "Porte Magique 3€99" ↓ Normalization: "Porte Magique 3.99 €" ↓ Extracted: "3€99" → Normalized: "3.99 €" ↓ Sent to AI: "PRIX DÉTECTÉ: 3.99 €\n\nPorte Magique 3.99 €" ↓ AI sees: "3.99" (NOT "399") ↓ Returns: 3.99 ✓ ``` **Expected Results:** - Before: "3€99" → AI returns 399.00 ❌ - After: "3€99" → Normalized to "3.99 €" → AI returns 3.99 ✅ Fixes L'Incroyable and any other sites using euro-as-separator format. Complements existing comma-separator normalization. --- app/services/browserless_service.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index 8e7b4c7..b1082ed 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -403,9 +403,11 @@ class BrowserlessService: content = await page.inner_text('body') logger.info(f"Extracted {len(content)} chars of visible text from page") - # Normalize French prices to English format for AI (3,99 € → 3.99 €) + # Normalize French prices to English format for AI import re - # Replace comma with dot in price patterns: digits,XX € → digits.XX € + # Pattern 1: digits€XX → digits.XX € (e.g., "3€99" → "3.99 €") + content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content) + # Pattern 2: digits,XX € → digits.XX € (e.g., "3,99 €" → "3.99 €") content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) # Remove spaces in thousands: 1 234.56 → 1234.56 content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content) @@ -432,11 +434,12 @@ class BrowserlessService: # Prepend extracted price to content if found if extracted_price: - # Convert French format to English for AI (3,99 → 3.99) + # Convert French format to English for AI import re normalized_price = extracted_price - # Replace comma with dot for decimal separator (French to English) - # Pattern: digits, comma, 2 digits → digits, dot, 2 digits + # Pattern 1: digits€digits → digits.digits € (e.g., "3€99" → "3.99 €") + normalized_price = re.sub(r'(\d+)€(\d{2})', r'\1.\2 €', normalized_price) + # Pattern 2: digits,digits → digits.digits (e.g., "3,99" → "3.99") normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price) # Remove spaces in thousands separators if any (1 234,56 → 1234.56) normalized_price = normalized_price.replace(' ', '')