From 3293b5d58a1918d597f10c46c25f825b0f2128e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 23 Nov 2025 08:12:43 +0000 Subject: [PATCH 1/2] fix: Improve French price extraction and cookie handling AI Schema: - Improve extraction prompt for French e-commerce pages - Add specific instructions for French price format (comma decimal, space thousands) - Add French stock status terms (Ajouter au panier, Disponible, etc.) - Fix price normalization to handle French format: - "12,99" -> 12.99 - "1 234,56" -> 1234.56 - "1.234,56" -> 1234.56 Scraper Service: - Add comprehensive cookie/popup selectors for French sites: - E.Leclerc, Auchan, Carrefour (OneTrust, Didomi) - Cdiscount (#footer_tc_privacy_button_2) - Boulanger (.bcom-consent-accept-all) - Gifi, Centrakor (Magento cookie buttons) - PrestaShop sites (Stokomani, B&M) - Add common CMP platform selectors - Add French text-based selectors for cookie acceptance --- app/ai_schema.py | 46 ++++++++++++++++++-------- app/services/scraper_service.py | 58 +++++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/app/ai_schema.py b/app/ai_schema.py index 7378917..1e7ad8d 100644 --- a/app/ai_schema.py +++ b/app/ai_schema.py @@ -73,14 +73,28 @@ class AIExtractionResponse(BaseModel): @field_validator("price", mode="before") @classmethod def normalize_price(cls, v): - """Normalize price to float or None.""" + """Normalize price to float or None, handling French format.""" if v is None or v in ("null", ""): return None if isinstance(v, str): - # Remove currency symbols and commas - cleaned = re.sub(r"[^\d.]", "", v) + # Handle French format: "1 234,56" -> "1234.56" + # First, remove spaces (thousand separators) + cleaned = v.replace(" ", "").replace("\u00a0", "") + # Replace comma with dot (French decimal separator) + # But only if there's no dot already (to handle "1.234,56" format) + if "," in cleaned and "." in cleaned: + # Format "1.234,56" - remove dots, replace comma with dot + cleaned = cleaned.replace(".", "").replace(",", ".") + elif "," in cleaned: + # Format "12,99" - just replace comma with dot + cleaned = cleaned.replace(",", ".") + # Remove currency symbols and other non-numeric chars (except dot) + cleaned = re.sub(r"[^\d.]", "", cleaned) if cleaned: - return float(cleaned) + try: + return float(cleaned) + except ValueError: + return None return None return float(v) @@ -116,23 +130,27 @@ class AIExtractionMetadata(BaseModel): # Prompt template for schema-first extraction -EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from the image. +EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from this French e-commerce page. -**PRICE:** -- Find the main current price -- Ignore crossed-out prices -- Extract number only (no symbols) +**PRICE (IMPORTANT - French format):** +- French prices use COMMA for decimals: "12,99 €" means 12.99 +- Thousands separator is SPACE or DOT: "1 234,56 €" or "1.234,56 €" means 1234.56 +- Currency symbol is usually € at the end +- Look for: price tags, "Prix:", "€", numbers near "Ajouter au panier" +- Extract as DECIMAL NUMBER (convert comma to dot): 12,99 -> 12.99 +- Ignore crossed-out/barré prices (old prices) +- If multiple prices, take the current/main price (not the original) - If unclear: set null and confidence < 0.5 **STOCK:** -- TRUE if: "Add to Cart", "Buy Now", "In Stock", "Available" -- FALSE if: "Out of Stock", "Sold Out", "Unavailable", "Notify Me" +- TRUE if: "Ajouter au panier", "Acheter", "En stock", "Disponible", "Add to Cart" +- FALSE if: "Rupture", "Indisponible", "Épuisé", "Out of Stock", "Notify Me" - NULL if unclear or not shown **CONFIDENCE (0.0 to 1.0):** -- 0.9-1.0: Very certain -- 0.5-0.8: Moderately certain -- Below 0.5: Unsure +- 0.9-1.0: Price clearly visible +- 0.5-0.8: Price found but partially obscured or uncertain +- Below 0.5: Cannot find price reliably Respond ONLY with valid JSON: {{ diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py index 1328bc8..059185b 100644 --- a/app/services/scraper_service.py +++ b/app/services/scraper_service.py @@ -209,16 +209,64 @@ class ScraperService: # Wait a bit for dynamic content if needed await page.wait_for_timeout(2000) - # Try to close common popups (inclut Amazon) - logger.info("Attempting to close popups...") + # Try to close common popups and cookie banners + logger.info("Attempting to close popups and cookie banners...") popup_selectors = [ - # Amazon spécifique - boutons cookies + # === SITES SPÉCIFIQUES FRANÇAIS === + # Amazon France "#sp-cc-accept", "#sp-cc-rejectall-link", "input[data-action-type='DISMISS']", "#a-popover-content-1 button", "[data-action='a-popover-close']", - # Popups généraux + # E.Leclerc, Darty, Fnac + "#onetrust-accept-btn-handler", + ".onetrust-accept-btn-handler", + # Auchan + "#popin_tc_privacy_button_2", + ".popin_tc_privacy_button", + "#didomi-notice-agree-button", + # Carrefour + "[data-testid='accept-cookies-button']", + # Cdiscount + "#footer_tc_privacy_button_2", + ".privacy_prompt_accept", + # Boulanger + ".bcom-consent-accept-all", + "#cookieBanner-accept", + # Gifi, Centrakor, La Foir'Fouille (Magento) + ".action-primary.action-accept", + "#btn-cookie-allow", + # PrestaShop (Stokomani, B&M, etc.) + ".btn-primary[data-dismiss='modal']", + "#gdpr_consent_agree", + + # === CMP (Consent Management Platforms) === + ".didomi-continue-without-agreeing", + "#tarteaucitronPersonalize2", + ".tarteaucitronAllow", + "#tarteaucitronAllDenied2", + ".cc-btn.cc-allow", + ".cky-btn-accept", + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", + "#CybotCookiebotDialogBodyButtonAccept", + ".qc-cmp2-summary-buttons button:first-child", + "#axeptio_btn_acceptAll", + ".axeptio_acceptAll", + + # === TEXTE FRANÇAIS === + "button:has-text('Tout accepter')", + "button:has-text('Accepter tout')", + "button:has-text('Accepter et fermer')", + "button:has-text('Accepter les cookies')", + "button:has-text('Accepter')", + "button:has-text('J\\'accepte')", + "button:has-text('OK')", + "button:has-text('Continuer')", + "a:has-text('Tout accepter')", + "a:has-text('Accepter')", + + # === POPUPS GÉNÉRAUX === "button[aria-label='Close']", "button[aria-label='close']", "button[aria-label='Fermer']", @@ -229,9 +277,7 @@ class ScraperService: "[class*='modal'] button", "button:has-text('No, thanks')", "button:has-text('No thanks')", - "button:has-text('Accepter')", "button:has-text('Accept')", - "button:has-text('Continuer')", "a:has-text('No, thanks')", "div[role='dialog'] button[aria-label='Close']", ] From 00d3aa9f1e32db3ce4b34e6360783be76145f271 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 23 Nov 2025 08:15:29 +0000 Subject: [PATCH 2/2] fix: Improve Amazon search with anti-bot bypass measures - Add special handling for Amazon (FR and US): - Custom User-Agent and HTTP headers mimicking real browser - Don't block images (Amazon uses them for bot detection) - Extra wait time and scroll for lazy-loaded content - Timezone set to Europe/Paris - Additional Sec-Fetch-* headers for authenticity - Update Amazon product selectors with more robust patterns: - a.a-link-normal for direct product links - [data-asin]:not([data-asin='']) for filtering empty ASINs - .s-main-slot targeting main search results - Add &ref=nb_sb_noss parameter to avoid redirects --- app/services/direct_search_service.py | 78 ++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py index 486e9c2..62eb042 100644 --- a/app/services/direct_search_service.py +++ b/app/services/direct_search_service.py @@ -166,15 +166,22 @@ DEFAULT_SITE_CONFIGS = { # === E-COMMERCE GÉNÉRALISTE === "amazon.fr": { "name": "Amazon France", - "search_url": "https://www.amazon.fr/s?k={query}&language=fr_FR", + # URL de recherche Amazon avec paramètres pour éviter les redirections + "search_url": "https://www.amazon.fr/s?k={query}&ref=nb_sb_noss", "product_selector": ( - "div[data-component-type='s-search-result'] h2 a, " - ".s-result-item h2 a, [data-asin] h2 a" + # Sélecteurs multiples pour robustesse + "div[data-component-type='s-search-result'] h2 a.a-link-normal, " + "div[data-asin] h2 a.a-link-normal, " + ".s-result-item h2 a.a-link-normal, " + "[data-asin]:not([data-asin='']) h2 a, " + ".s-main-slot .s-result-item a.a-link-normal.s-no-outline" ), - "wait_selector": "div[data-component-type='s-search-result'], .s-result-item", + "wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item", "category": "E-commerce", "requires_js": True, "priority": 11, + # Amazon nécessite une attention particulière pour les cookies + "needs_cookie_accept": True, }, "cdiscount.com": { "name": "Cdiscount", @@ -216,12 +223,18 @@ DEFAULT_SITE_CONFIGS = { # === AUTRES (conservés pour compatibilité) === "amazon.com": { "name": "Amazon US", - "search_url": "https://www.amazon.com/s?k={query}", - "product_selector": "div[data-component-type='s-search-result'] h2 a", - "wait_selector": "div[data-component-type='s-search-result']", + "search_url": "https://www.amazon.com/s?k={query}&ref=nb_sb_noss", + "product_selector": ( + "div[data-component-type='s-search-result'] h2 a.a-link-normal, " + "div[data-asin] h2 a.a-link-normal, " + ".s-result-item h2 a.a-link-normal, " + "[data-asin]:not([data-asin='']) h2 a" + ), + "wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item", "category": "E-commerce", "requires_js": True, "priority": 99, + "needs_cookie_accept": True, }, "fnac.com": { "name": "Fnac", @@ -436,21 +449,51 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 logger.info(f"Recherche Browserless sur {domain}: {final_url}") + # Détecter si c'est Amazon (protection anti-bot forte) + is_amazon = "amazon" in domain + try: async with async_playwright() as p: # Se connecter à Browserless browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL) try: - context = await browser.new_context( - viewport={"width": 1920, "height": 1080}, - user_agent=USER_AGENT, - locale="fr-FR", - ) + # Configuration spéciale pour Amazon + if is_amazon: + context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + locale="fr-FR", + timezone_id="Europe/Paris", + extra_http_headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + }, + ) + else: + context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=USER_AGENT, + locale="fr-FR", + ) + page = await context.new_page() - # Bloquer les ressources inutiles pour accélérer - await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) + # Pour Amazon, ne pas bloquer les images (détection anti-bot) + if not is_amazon: + await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) + await page.route("**/analytics*", lambda route: route.abort()) await page.route("**/tracking*", lambda route: route.abort()) @@ -460,6 +503,13 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 # Accepter les cookies si nécessaire await _accept_cookies(page, domain) + # Pour Amazon, attendre plus longtemps et scroller + if is_amazon: + await asyncio.sleep(3) + # Scroll pour charger le contenu lazy-loaded + await page.evaluate("window.scrollBy(0, 500)") + await asyncio.sleep(1) + # Attendre que les résultats se chargent if wait_selector: try: