Merge pull request #34 from R0m1k3/claude/hardcode-website-urls-01QrWZK7suQnuBrxYgK1r5JC

Claude/hardcode website urls 01 qr wzk7su qnu brx yg k1r5 jc

app/ai_schema.py

@@ -73,14 +73,28 @@ class AIExtractionResponse(BaseModel):
     @field_validator("price", mode="before")
     @classmethod
     def normalize_price(cls, v):
-        """Normalize price to float or None."""
+        """Normalize price to float or None, handling French format."""
         if v is None or v in ("null", ""):
             return None
         if isinstance(v, str):
-            # Remove currency symbols and commas
+            # Handle French format: "1 234,56" -> "1234.56"
-            cleaned = re.sub(r"[^\d.]", "", v)
+            # First, remove spaces (thousand separators)
+            cleaned = v.replace(" ", "").replace("\u00a0", "")
+            # Replace comma with dot (French decimal separator)
+            # But only if there's no dot already (to handle "1.234,56" format)
+            if "," in cleaned and "." in cleaned:
+                # Format "1.234,56" - remove dots, replace comma with dot
+                cleaned = cleaned.replace(".", "").replace(",", ".")
+            elif "," in cleaned:
+                # Format "12,99" - just replace comma with dot
+                cleaned = cleaned.replace(",", ".")
+            # Remove currency symbols and other non-numeric chars (except dot)
+            cleaned = re.sub(r"[^\d.]", "", cleaned)
             if cleaned:
-                return float(cleaned)
+                try:
+                    return float(cleaned)
+                except ValueError:
+                    return None
             return None
         return float(v)
 
@@ -116,23 +130,27 @@ class AIExtractionMetadata(BaseModel):
 
 
 # Prompt template for schema-first extraction
-EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from the image.
+EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from this French e-commerce page.
 
-**PRICE:**
+**PRICE (IMPORTANT - French format):**
-- Find the main current price
+- French prices use COMMA for decimals: "12,99 €" means 12.99
-- Ignore crossed-out prices
+- Thousands separator is SPACE or DOT: "1 234,56 €" or "1.234,56 €" means 1234.56
-- Extract number only (no symbols)
+- Currency symbol is usually € at the end
+- Look for: price tags, "Prix:", "€", numbers near "Ajouter au panier"
+- Extract as DECIMAL NUMBER (convert comma to dot): 12,99 -> 12.99
+- Ignore crossed-out/barré prices (old prices)
+- If multiple prices, take the current/main price (not the original)
 - If unclear: set null and confidence < 0.5
 
 **STOCK:**
-- TRUE if: "Add to Cart", "Buy Now", "In Stock", "Available"
+- TRUE if: "Ajouter au panier", "Acheter", "En stock", "Disponible", "Add to Cart"
-- FALSE if: "Out of Stock", "Sold Out", "Unavailable", "Notify Me"
+- FALSE if: "Rupture", "Indisponible", "Épuisé", "Out of Stock", "Notify Me"
 - NULL if unclear or not shown
 
 **CONFIDENCE (0.0 to 1.0):**
-- 0.9-1.0: Very certain
+- 0.9-1.0: Price clearly visible
-- 0.5-0.8: Moderately certain
+- 0.5-0.8: Price found but partially obscured or uncertain
-- Below 0.5: Unsure
+- Below 0.5: Cannot find price reliably
 
 Respond ONLY with valid JSON:
 {{

app/services/direct_search_service.py

@@ -166,15 +166,22 @@ DEFAULT_SITE_CONFIGS = {
     # === E-COMMERCE GÉNÉRALISTE ===
     "amazon.fr": {
         "name": "Amazon France",
-        "search_url": "https://www.amazon.fr/s?k={query}&language=fr_FR",
+        # URL de recherche Amazon avec paramètres pour éviter les redirections
+        "search_url": "https://www.amazon.fr/s?k={query}&ref=nb_sb_noss",
         "product_selector": (
-            "div[data-component-type='s-search-result'] h2 a, "
+            # Sélecteurs multiples pour robustesse
-            ".s-result-item h2 a, [data-asin] h2 a"
+            "div[data-component-type='s-search-result'] h2 a.a-link-normal, "
+            "div[data-asin] h2 a.a-link-normal, "
+            ".s-result-item h2 a.a-link-normal, "
+            "[data-asin]:not([data-asin='']) h2 a, "
+            ".s-main-slot .s-result-item a.a-link-normal.s-no-outline"
         ),
-        "wait_selector": "div[data-component-type='s-search-result'], .s-result-item",
+        "wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item",
         "category": "E-commerce",
         "requires_js": True,
         "priority": 11,
+        # Amazon nécessite une attention particulière pour les cookies
+        "needs_cookie_accept": True,
     },
     "cdiscount.com": {
         "name": "Cdiscount",
@@ -216,12 +223,18 @@ DEFAULT_SITE_CONFIGS = {
     # === AUTRES (conservés pour compatibilité) ===
     "amazon.com": {
         "name": "Amazon US",
-        "search_url": "https://www.amazon.com/s?k={query}",
+        "search_url": "https://www.amazon.com/s?k={query}&ref=nb_sb_noss",
-        "product_selector": "div[data-component-type='s-search-result'] h2 a",
+        "product_selector": (
-        "wait_selector": "div[data-component-type='s-search-result']",
+            "div[data-component-type='s-search-result'] h2 a.a-link-normal, "
+            "div[data-asin] h2 a.a-link-normal, "
+            ".s-result-item h2 a.a-link-normal, "
+            "[data-asin]:not([data-asin='']) h2 a"
+        ),
+        "wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item",
         "category": "E-commerce",
         "requires_js": True,
         "priority": 99,
+        "needs_cookie_accept": True,
     },
     "fnac.com": {
         "name": "Fnac",
@@ -436,21 +449,51 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
 
     logger.info(f"Recherche Browserless sur {domain}: {final_url}")
 
+    # Détecter si c'est Amazon (protection anti-bot forte)
+    is_amazon = "amazon" in domain
+
     try:
         async with async_playwright() as p:
             # Se connecter à Browserless
             browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
 
             try:
-                context = await browser.new_context(
+                # Configuration spéciale pour Amazon
-                    viewport={"width": 1920, "height": 1080},
+                if is_amazon:
-                    user_agent=USER_AGENT,
+                    context = await browser.new_context(
-                    locale="fr-FR",
+                        viewport={"width": 1920, "height": 1080},
-                )
+                        user_agent=(
+                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                            "AppleWebKit/537.36 (KHTML, like Gecko) "
+                            "Chrome/120.0.0.0 Safari/537.36"
+                        ),
+                        locale="fr-FR",
+                        timezone_id="Europe/Paris",
+                        extra_http_headers={
+                            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                            "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
+                            "Accept-Encoding": "gzip, deflate, br",
+                            "Connection": "keep-alive",
+                            "Upgrade-Insecure-Requests": "1",
+                            "Sec-Fetch-Dest": "document",
+                            "Sec-Fetch-Mode": "navigate",
+                            "Sec-Fetch-Site": "none",
+                            "Sec-Fetch-User": "?1",
+                        },
+                    )
+                else:
+                    context = await browser.new_context(
+                        viewport={"width": 1920, "height": 1080},
+                        user_agent=USER_AGENT,
+                        locale="fr-FR",
+                    )
+
                 page = await context.new_page()
 
-                # Bloquer les ressources inutiles pour accélérer
+                # Pour Amazon, ne pas bloquer les images (détection anti-bot)
-                await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+                if not is_amazon:
+                    await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+
                 await page.route("**/analytics*", lambda route: route.abort())
                 await page.route("**/tracking*", lambda route: route.abort())
 
@@ -460,6 +503,13 @@ async def _search_site_browserless(  # noqa: PLR0912, PLR0915
                 # Accepter les cookies si nécessaire
                 await _accept_cookies(page, domain)
 
+                # Pour Amazon, attendre plus longtemps et scroller
+                if is_amazon:
+                    await asyncio.sleep(3)
+                    # Scroll pour charger le contenu lazy-loaded
+                    await page.evaluate("window.scrollBy(0, 500)")
+                    await asyncio.sleep(1)
+
                 # Attendre que les résultats se chargent
                 if wait_selector:
                     try:

app/services/scraper_service.py

@@ -209,16 +209,64 @@ class ScraperService:
                 # Wait a bit for dynamic content if needed
                 await page.wait_for_timeout(2000)
 
-                # Try to close common popups (inclut Amazon)
+                # Try to close common popups and cookie banners
-                logger.info("Attempting to close popups...")
+                logger.info("Attempting to close popups and cookie banners...")
                 popup_selectors = [
-                    # Amazon spécifique - boutons cookies
+                    # === SITES SPÉCIFIQUES FRANÇAIS ===
+                    # Amazon France
                     "#sp-cc-accept",
                     "#sp-cc-rejectall-link",
                     "input[data-action-type='DISMISS']",
                     "#a-popover-content-1 button",
                     "[data-action='a-popover-close']",
-                    # Popups généraux
+                    # E.Leclerc, Darty, Fnac
+                    "#onetrust-accept-btn-handler",
+                    ".onetrust-accept-btn-handler",
+                    # Auchan
+                    "#popin_tc_privacy_button_2",
+                    ".popin_tc_privacy_button",
+                    "#didomi-notice-agree-button",
+                    # Carrefour
+                    "[data-testid='accept-cookies-button']",
+                    # Cdiscount
+                    "#footer_tc_privacy_button_2",
+                    ".privacy_prompt_accept",
+                    # Boulanger
+                    ".bcom-consent-accept-all",
+                    "#cookieBanner-accept",
+                    # Gifi, Centrakor, La Foir'Fouille (Magento)
+                    ".action-primary.action-accept",
+                    "#btn-cookie-allow",
+                    # PrestaShop (Stokomani, B&M, etc.)
+                    ".btn-primary[data-dismiss='modal']",
+                    "#gdpr_consent_agree",
+
+                    # === CMP (Consent Management Platforms) ===
+                    ".didomi-continue-without-agreeing",
+                    "#tarteaucitronPersonalize2",
+                    ".tarteaucitronAllow",
+                    "#tarteaucitronAllDenied2",
+                    ".cc-btn.cc-allow",
+                    ".cky-btn-accept",
+                    "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
+                    "#CybotCookiebotDialogBodyButtonAccept",
+                    ".qc-cmp2-summary-buttons button:first-child",
+                    "#axeptio_btn_acceptAll",
+                    ".axeptio_acceptAll",
+
+                    # === TEXTE FRANÇAIS ===
+                    "button:has-text('Tout accepter')",
+                    "button:has-text('Accepter tout')",
+                    "button:has-text('Accepter et fermer')",
+                    "button:has-text('Accepter les cookies')",
+                    "button:has-text('Accepter')",
+                    "button:has-text('J\\'accepte')",
+                    "button:has-text('OK')",
+                    "button:has-text('Continuer')",
+                    "a:has-text('Tout accepter')",
+                    "a:has-text('Accepter')",
+
+                    # === POPUPS GÉNÉRAUX ===
                     "button[aria-label='Close']",
                     "button[aria-label='close']",
                     "button[aria-label='Fermer']",
@@ -229,9 +277,7 @@ class ScraperService:
                     "[class*='modal'] button",
                     "button:has-text('No, thanks')",
                     "button:has-text('No thanks')",
-                    "button:has-text('Accepter')",
                     "button:has-text('Accept')",
-                    "button:has-text('Continuer')",
                     "a:has-text('No, thanks')",
                     "div[role='dialog'] button[aria-label='Close']",
                 ]