feat: Add initial database migration and UI label component, alongside frontend dependency installation.

app/services/direct_search_service.py

@@ -70,8 +70,10 @@ DEFAULT_SITE_CONFIGS = {
         "name": "B&M",
         "search_url": "https://bmstores.fr/module/ambjolisearch/jolisearch?s={query}",
         "product_selector": (
-            ".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
-            ".products article a, a[href*='/produits/']"
+            # Cibler uniquement les liens produits, pas les catégories
+            "a.product-thumbnail[href*='/produits/']:not([href*='/c/']):not([href*='/categorie']), "
+            ".product-miniature a[href*='/produits/']:not([href*='/c/']), "
+            ".js-product-miniature a[href*='/produits/']"
         ),
         "wait_selector": ".products, .product-miniature, #js-product-list, #search_results",
         "category": "Discount",
@@ -190,10 +192,12 @@ DEFAULT_SITE_CONFIGS = {
         "name": "Cdiscount",
         "search_url": "https://www.cdiscount.com/search/10/{query}.html",
         "product_selector": (
-            "a.prdtBImg, .prdtBILDetails a, .product-item a, "
-            "a[href*='/f-']"
+            # Sélecteurs spécifiques Cdiscount - produits uniquement
+            "a.prdtBImg[href*='/f-'][href*='.html'], "
+            ".prdtBILDetails a[href*='/f-'][href*='.html'], "
+            "a[href*='/f-'][href*='.html']:not([href*='/l-']):not([href*='/rayon'])"
         ),
-        "wait_selector": ".prdtBILDetails, .product-list",
+        "wait_selector": ".prdtBILDetails, .product-list, #lpBloc",
         "category": "E-commerce",
         "requires_js": True,
         "priority": 12,
@@ -203,10 +207,12 @@ DEFAULT_SITE_CONFIGS = {
         "name": "Darty",
         "search_url": "https://www.darty.com/nav/recherche?text={query}",
         "product_selector": (
-            ".product-card a, a[href*='/product'], "
-            ".product-list-item a"
+            # Cibler les fiches produits, exclure navigation et catégories
+            "a[href*='/nav/achat/'][href*='.html']:not([href*='/rayon']):not([href*='/promo']), "
+            ".product-card a[href*='/nav/achat/'], "
+            "a.product-link[href*='/nav/achat/']"
         ),
-        "wait_selector": ".product-card, .product-list",
+        "wait_selector": ".product-card, .product-list, .product-item",
         "category": "Électronique",
         "requires_js": True,
         "priority": 13,
@@ -215,10 +221,12 @@ DEFAULT_SITE_CONFIGS = {
         "name": "Boulanger",
         "search_url": "https://www.boulanger.com/resultats?tr={query}",
         "product_selector": (
-            ".product-list__item a, a[href*='/ref/'], "
-            ".product-card a"
+            # Cibler uniquement les pages produits avec /ref/
+            "a[href*='/ref/'][href*='_']:not([href*='/c/']):not([href*='/rayon']), "
+            ".product-list__item a[href*='/ref/'], "
+            ".product-card__link[href*='/ref/']"
         ),
-        "wait_selector": ".product-list, .product-card",
+        "wait_selector": ".product-list, .product-card, .products-list",
         "category": "Électronique",
         "requires_js": True,
         "priority": 14,
@@ -241,8 +249,13 @@ DEFAULT_SITE_CONFIGS = {
     "fnac.com": {
         "name": "Fnac",
         "search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
-        "product_selector": "a.Article-title, .Article-item a.js-minifa-title",
-        "wait_selector": ".Article-item",
+        "product_selector": (
+            # Sélecteurs Fnac pour les produits
+            "a.Article-title[href*='/a'], "
+            ".Article-item a.js-minifa-title[href*='/a'], "
+            "a[href*='/a'][href*='.html']:not([href*='/rayon']):not([href*='/univers'])"
+        ),
+        "wait_selector": ".Article-item, .Article-list, #SearchResultList",
         "category": "Culture & Tech",
         "requires_js": True,
         "priority": 15,
@@ -721,8 +734,9 @@ def _is_navigation_link(href: str) -> bool:
     """Vérifie si un lien est un lien de navigation (pas un produit)"""
     nav_patterns = [
         # Catégories et navigation
-        "/category", "/categories", "/categorie",
-        "/brand", "/brands", "/marque",
+        "/category", "/categories", "/categorie", "/c/",
+        "/rayon", "/rayons", "/univers",
+        "/brand", "/brands", "/marque", "/marques",
         "/page/", "/pages/",
         "/cart", "/panier", "/basket",
         "/checkout", "/commande",
@@ -740,13 +754,24 @@ def _is_navigation_link(href: str) -> bool:
         "/application", "/app/", "/apps/",
         "/courses", "/course/",
         "/ep-", "/endpoint",
+        # Promotions et offres (patterns communs)
         "/promo/", "/promotions/", "/promotion/",
-        "/offres/", "/offre/",
+        "/soldes/", "/solde/",
+        "/bon-plan", "/bons-plans",
+        "/offres/", "/offre/", "/offre-",
+        "/deals/", "/deal/",
+        "/ventes-flash", "/vente-flash",
+        # Services et pages informatives
         "/services/", "/service/",
         "/magasin", "/magasins", "/stores/", "/store/",
         "/blog", "/actualites", "/news",
         "/recettes", "/recipes",
         "/conseils", "/tips",
+        "/guide", "/guides",
+        # Patterns e-commerce spécifiques
+        "/l-", "/lp-", "/landing",  # Landing pages (Cdiscount, etc.)
+        "/selection", "/selections",
+        "/top-", "/best-",
         # Amazon spécifique
         "/ref=cs_", "/logo", "/nav_", "/gp/help",
         "/gp/css", "/gp/redirect", "/hz/",
@@ -763,6 +788,7 @@ def _is_navigation_link(href: str) -> bool:
     return any(pattern in href_lower for pattern in nav_patterns)
 
 
+
 def _extract_title(element) -> str:
     """Extrait le titre d'un élément"""
     # Essayer différentes méthodes
@@ -855,16 +881,28 @@ def _is_non_product_url(url: str) -> bool:
                 return True
 
     non_product_patterns = [
-        "/search", "/category", "/categories", "/brand", "/brands",
+        # Navigation et catégories
+        "/search", "/category", "/categories", "/categorie", "/c/",
+        "/rayon", "/rayons", "/univers",
+        "/brand", "/brands", "/marque", "/marques",
+        # Pages utilisateur
         "/help", "/contact", "/about", "/account", "/cart", "/checkout",
         "/login", "/register", "/wishlist", "/compare", "/reviews",
+        # Pages légales
         "/terms", "/privacy", "/faq", "/shipping", "/returns",
+        # Erreurs
         "/503", "/error", "/robot", "/captcha",
+        # Applications et services
         "/application", "/courses", "/app/",
+        # Landing pages et promotions (patterns spécifiques e-commerce français)
+        "/l-", "/lp-", "/landing",  # Cdiscount landing pages
+        "/promo/", "/soldes/", "/bon-plan", "/offre-", "/deals/",
+        "/selection", "/top-", "/best-",
     ]
     return any(pattern in url_lower for pattern in non_product_patterns)
 
 
+
 async def health_check() -> bool:
     """Vérifie la connectivité avec Browserless"""
     try:

app/services/scheduler_service.py

@@ -103,7 +103,7 @@ async def process_item_check(item_id: int):
 
     try:
         logger.info(f"Checking item: {item_data['name']} ({item_data['url']})")
-        screenshot_path, page_text = await ScraperService.scrape_item(
+        screenshot_path, page_text, is_available = await ScraperService.scrape_item(
             item_data["url"],
             item_data["selector"],
             item_id,
@@ -113,6 +113,15 @@ async def process_item_check(item_id: int):
             timeout=config["scraper_timeout"],
         )
 
+        # Update availability status in database
+        with database.SessionLocal() as session:
+            if item := session.query(models.Item).filter(models.Item.id == item_id).first():
+                item.is_available = is_available
+                if not is_available:
+                    logger.warning(f"Product {item_id} marked as unavailable")
+                    item.last_error = "Product no longer available (404 or not found)"
+                session.commit()
+
         if not screenshot_path:
             raise Exception("Failed to capture screenshot")
 

app/services/scraper_service.py

@@ -88,10 +88,10 @@ class ScraperService:
         scroll_pixels: int = 350,
         text_length: int = 0,
         timeout: int = 90000,
-    ) -> tuple[str | None, str]:
+    ) -> tuple[str | None, str, bool]:
         """
         Scrapes the given URL using Browserless and Playwright.
-        Returns a tuple: (screenshot_path, page_text)
+        Returns a tuple: (screenshot_path, page_text, is_available)
 
         Args:
             url: Target URL to scrape
@@ -139,7 +139,7 @@ class ScraperService:
             logger.warning(f"Tentative {attempt + 1} échouée pour {url}")
 
         logger.error(f"Toutes les tentatives ont échoué pour {url}")
-        return None, ""
+        return None, "", False  # Product unavailable if all retries failed
 
     @staticmethod
     async def _do_scrape(  # noqa: PLR0913, PLR0912, PLR0915
@@ -150,12 +150,14 @@ class ScraperService:
         scroll_pixels: int = 350,
         text_length: int = 0,
         timeout: int = 90000,
-    ) -> tuple[str | None, str]:
+    ) -> tuple[str | None, str, bool]:
         """Exécute le scraping réel (appelé par scrape_item avec retries)."""
         async with async_playwright() as p:
             browser = None
             context = None
             page = None
+            is_available = True  # Par défaut, le produit est disponible
+            
             try:
                 logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
 
@@ -188,9 +190,24 @@ class ScraperService:
                 page = await context.new_page()
 
                 logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
+                response = None
                 try:
                     # First wait for domcontentloaded - this is the minimum we need
-                    await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
+                    response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
+                    
+                    # Check HTTP status code
+                    if response:
+                        status = response.status
+                        logger.info(f"HTTP Status: {status}")
+                        
+                        # Detect unavailable products by HTTP status
+                        if status in [404, 410, 451, 503]:
+                            logger.warning(f"Product unavailable - HTTP {status}")
+                            is_available = False
+                        elif status >= 400:
+                            logger.warning(f"HTTP error {status}, marking as potentially unavailable")
+                            is_available = False
+                    
                     logger.info(f"Page loaded (domcontentloaded): {url}")
 
                     # Then try to wait for networkidle, but don't fail if it times out
@@ -203,6 +220,8 @@ class ScraperService:
 
                 except Exception as e:
                     logger.error(f"Error navigating to {url}: {e}")
+                    # If navigation fails completely, mark as unavailable
+                    is_available = False
                     # Try to take screenshot anyway if page partially loaded
                     pass
 
@@ -347,7 +366,7 @@ class ScraperService:
                     logger.info(f"Screenshot saved to {filename}")
                 except Exception as e:
                     logger.error(f"Screenshot failed: {e}")
-                    return None, ""
+                    return None, "", is_available
 
                 # Text Extraction (après le screenshot pour ne pas le perdre)
                 page_text = ""
@@ -359,15 +378,40 @@ class ScraperService:
                         # Simple truncation
                         page_text = raw_text[:text_length]
                         logger.info(f"Extracted {len(page_text)} characters")
+                        
+                        # Check for "product not found" messages in page text
+                        unavailable_patterns = [
+                            "produit introuvable",
+                            "produit indisponible",
+                            "page introuvable",
+                            "404",
+                            "product not found",
+                            "item not found",
+                            "page not found",
+                            "n'existe plus",
+                            "plus disponible",
+                            "no longer available",
+                            "article introuvable",
+                            "cette page n'existe pas",
+                            "this page doesn't exist",
+                        ]
+                        
+                        page_text_lower = page_text.lower()
+                        for pattern in unavailable_patterns:
+                            if pattern in page_text_lower:
+                                logger.warning(f"Product unavailable - found pattern: '{pattern}'")
+                                is_available = False
+                                break
+                                
                     except Exception as e:
                         logger.warning(f"Text extraction failed (screenshot saved): {e}")
                         # On continue car le screenshot a été pris
 
-                return filename, page_text
+                return filename, page_text, is_available
 
             except Exception as e:
                 logger.error(f"Error scraping {url}: {e}")
-                return None, ""
+                return None, "", False  # Mark as unavailable on error
 
             finally:
                 # Fermeture propre des ressources dans l'ordre inverse

app/services/search_service.py

@@ -209,7 +209,7 @@ async def _scrape_with_browserless(
     """Scrape une URL avec Browserless + extraction IA"""
     try:
         # Scraper avec Browserless
-        screenshot_path, page_text = await ScraperService.scrape_item(
+        screenshot_path, page_text, is_available = await ScraperService.scrape_item(
             url=url,
             item_id=None,
             smart_scroll=True,