From 531ed1bc15d6aef7a98d06b8ef56b5e7de2bf13a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 23:51:00 +0000 Subject: [PATCH] fix: Convert French price format to English before sending to AI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem:** AI was misinterpreting French decimal comma as thousands separator: - "3,99 €" detected as 399.00 EUR (instead of 3.99 EUR) - "1,99 €" detected as 199.00 EUR (instead of 1.99 EUR) - Caused by AI reading comma as grouping separator, not decimal point **Root Cause:** French format uses comma for decimals: "3,99 €" = three euros ninety-nine cents English format uses dot for decimals: "3.99 €" = three euros ninety-nine cents AI models (trained mostly on English) interpret: - "3,99" → "3" and "99" separate → 399 (three hundred ninety-nine) - Should be: "3.99" → 3.99 (three point ninety-nine) **Solution: Pre-Convert Format Before AI Processing** 1. **Normalize ALL Text** (browserless_service.py:380-386) ```python # Convert French to English in all page text content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) # "3,99 €" → "3.99 €" # "12,50 €" → "12.50 €" # "1 234,56 €" → "1234.56 €" (also removes space thousands separator) ``` 2. **Normalize Extracted Price** (browserless_service.py:400-410) ```python # Convert French format in directly extracted price normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', extracted_price) # "3,99 €" → "3.99 €" # Prepends: "PRIX DÉTECTÉ: 3.99 €" (not "3,99 €") ``` 3. **Updated AI Prompt** (ai_schema.py:135-158) - Removed instructions to convert comma→dot (now done automatically) - Added: "The price has already been converted to English format for you" - Added explicit warnings about common mistakes: * "3.99 €" means 3.99 (NOT 399.00, NOT 3990.00) * "1.99 €" means 1.99 (NOT 199.00, NOT 1990.00) - Clear examples with both CORRECT and INCORRECT values **Flow:** ``` Page text: "Lutin 3,99 € Ajouter au panier" ↓ Normalization: "Lutin 3.99 € Ajouter au panier" ↓ Extracted price: "3,99 €" → Normalized: "3.99 €" ↓ Sent to AI: "PRIX DÉTECTÉ: 3.99 €\n\nLutin 3.99 € Ajouter au panier" ↓ AI sees only English format: "3.99" ↓ AI Response: {"price": 3.99} ✅ CORRECT ``` **Expected Results:** - Before: "3,99 €" → AI returns 399.00 ❌ - After: "3,99 €" → Normalized to "3.99 €" → AI returns 3.99 ✅ **Regex Details:** - `(\d+),(\d{2})\s*€` matches French prices: captures digits before comma, 2 digits after - `\1.\2 €` replaces with: same digits, dot instead of comma, space + euro symbol - Handles: 0,99 / 3,99 / 12,99 / 89,99 / 1234,99 Fixes issue where AI misread French decimal comma causing 100x price errors. --- app/ai_schema.py | 28 +++++++++++++++------------- app/services/browserless_service.py | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/app/ai_schema.py b/app/ai_schema.py index 65621d1..1500623 100644 --- a/app/ai_schema.py +++ b/app/ai_schema.py @@ -134,26 +134,28 @@ EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from this **PRICE (IMPORTANT - French format):** - **FIRST**: Look for "PRIX DÉTECTÉ:" at the start of text - this is the extracted price -- French prices use COMMA for decimals: "12,99 €" means 12.99 -- Thousands separator is SPACE or DOT: "1 234,56 €" or "1.234,56 €" means 1234.56 +- **The price has already been converted to English format for you** +- French original: "3,99 €" → Already shown to you as: "3.99 €" +- Just extract the number you see (e.g., "3.99" from "PRIX DÉTECTÉ: 3.99 €") - Currency symbol is usually € at the end - Look for: "PRIX DÉTECTÉ:", price tags, "Prix:", "€", numbers near "Ajouter au panier" -- Extract as DECIMAL NUMBER (convert comma to dot): 12,99 -> 12.99 +- Extract as DECIMAL NUMBER: If you see "3.99", return 3.99 - Ignore crossed-out/barré prices (old prices) - If multiple prices, take the current/main price (not the original) -**CRITICAL - Read digits carefully:** -- PAY CLOSE ATTENTION to first digit: "1,99 €" is NOT "9,99 €" -- Double-check: Is it 1, 7, 8, or 9? These look similar -- Small prices (< 5€) are common: 0,99 €, 1,49 €, 1,99 €, 2,99 €, 3,99 €, 4,99 € -- VERIFY the price makes sense for the product type +**CRITICAL - Common mistakes to avoid:** +- "3.99 €" means 3.99 (NOT 399.00, NOT 3990.00) +- "1.99 €" means 1.99 (NOT 199.00, NOT 1990.00) +- "0.99 €" means 0.99 (NOT 99.00, NOT 990.00) +- The decimal point separates euros from cents +- Small prices (< 10€) are very common for everyday items - Examples of valid prices: - * "1,99 €" -> 1.99 (NOT 9.99) - * "0,99 €" -> 0.99 (NOT 9.99) - * "89,99 €" -> 89.99 - * "1 234,56 €" -> 1234.56 - * "PRIX DÉTECTÉ: 89,99 €" -> 89.99 + * "PRIX DÉTECTÉ: 1.99 €" -> 1.99 (NOT 199 or 1990) + * "PRIX DÉTECTÉ: 3.99 €" -> 3.99 (NOT 399 or 3990) + * "0.99 €" -> 0.99 (NOT 99) + * "89.99 €" -> 89.99 (NOT 8999) + * "1234.56 €" -> 1234.56 - If you find ANY price with € symbol, extract it with confidence >= 0.8 - If digits are unclear or blurry, reduce confidence to 0.5-0.7 - If unclear: set null and confidence < 0.5 diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py index a55e98b..708a4b1 100644 --- a/app/services/browserless_service.py +++ b/app/services/browserless_service.py @@ -376,6 +376,15 @@ class BrowserlessService: try: content = await page.inner_text('body') logger.info(f"Extracted {len(content)} chars of visible text from page") + + # Normalize French prices to English format for AI (3,99 € → 3.99 €) + import re + # Replace comma with dot in price patterns: digits,XX € → digits.XX € + content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) + # Remove spaces in thousands: 1 234.56 → 1234.56 + content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content) + logger.debug("Normalized French price formats to English") + except Exception as e: # Fallback to HTML content if inner_text fails logger.warning(f"Failed to extract inner_text, falling back to HTML: {e}") @@ -397,8 +406,17 @@ class BrowserlessService: # Prepend extracted price to content if found if extracted_price: - content = f"PRIX DÉTECTÉ: {extracted_price}\n\n{content}" - logger.info(f"Prepended price to content: {extracted_price}") + # Convert French format to English for AI (3,99 → 3.99) + import re + normalized_price = extracted_price + # Replace comma with dot for decimal separator (French to English) + # Pattern: digits, comma, 2 digits → digits, dot, 2 digits + normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price) + # Remove spaces in thousands separators if any (1 234,56 → 1234.56) + normalized_price = normalized_price.replace(' ', '') + + content = f"PRIX DÉTECTÉ: {normalized_price}\n\n{content}" + logger.info(f"Prepended normalized price to content: {normalized_price} (original: {extracted_price})") else: logger.warning("Could not extract price from DOM, relying on text/image only") else: