feat: add search_config module for browserless, proxy, user agent, and site-specific search configurations, and create list_missing_sites script.

app/core/search_config.py

@@ -1,9 +1,5 @@
 """
-Search Configuration
-Centralizes all configuration for the search system:
-- Site definitions (selectors, URLs)
-- Proxy configurations
-- User Agents
+Search Configuration - COMPLETE FIXED VERSION
 """
 
 import os
@@ -14,7 +10,6 @@ BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 DEBUG_DUMPS_DIR = "/app/debug_dumps"
 
 # === PROXY CONFIGURATION ===
-# Format from user: ip:port:username:password
 AMAZON_PROXY_LIST_RAW = [
     "142.111.48.253:7030:jasuwwjr:elbsx170nmnl",
     "31.59.20.176:6754:jasuwwjr:elbsx170nmnl",
@@ -43,15 +38,11 @@ def get_amazon_proxies() -> list[dict]:
 
 # === USER AGENTS ===
 USER_AGENT_POOL = [
-    # Chrome Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    # Chrome Mac
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    # Firefox Windows
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
-    # Recent Chrome
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
 ]
@@ -89,6 +80,15 @@ SITE_CONFIGS = {
         "category": "Discount",
         "requires_proxy": False,
     },
+    "lafoirfouille.fr": {
+        "name": "La Foir'Fouille",
+        "search_url": "https://www.lafoirfouille.fr/recherche?controller=search&s={query}",
+        "product_selector": "article.product-miniature a.product-thumbnail, a.product_img_link",
+        "product_image_selector": "img[src], img[loading='lazy'], picture img",
+        "wait_selector": "article.product-miniature, .products",
+        "category": "Discount",
+        "requires_proxy": False,
+    },
     "cdiscount.com": {
         "name": "Cdiscount",
         "search_url": "https://www.cdiscount.com/search/10/{query}.html",
@@ -186,41 +186,37 @@ SITE_CONFIGS = {
 
 # === COOKIE BANNERS ===
 COOKIE_ACCEPT_SELECTORS = [
-    "#sp-cc-accept", # Amazon
-    "#onetrust-accept-btn-handler", # Many sites
+    "#sp-cc-accept",
+    "#onetrust-accept-btn-handler",
     ".onetrust-accept-btn-handler",
-    "#popin_tc_privacy_button_2", # Auchan/Cdiscount
+    "#popin_tc_privacy_button_2",
     ".popin_tc_privacy_button",
-    "#didomi-notice-agree-button", # Many French sites
+    "#didomi-notice-agree-button",
     ".didomi-continue-without-agreeing",
-    "[data-testid='accept-cookies-button']", # Carrefour
-    ".bcom-consent-accept-all", # Boulanger
+    "[data-testid='accept-cookies-button']",
+    ".bcom-consent-accept-all",
     "#cookieBanner-accept",
-    ".action-primary.action-accept", # Magento
+    ".action-primary.action-accept",
     "#btn-cookie-allow",
     "button:has-text('Tout accepter')",
     "button:has-text('Accepter tout')",
     "button:has-text('Accepter')",
     "button:has-text('J\\'accepte')",
-    # Promotional Popups / Modals
     ".modal-close",
     ".close-modal",
     ".popup-close",
-    "button[aria-label='Close']",
+   "button[aria-label='Close']",
     "button[aria-label='Fermer']",
     ".js-modal-close",
     "div[class*='popup'] button[class*='close']",
     "div[class*='modal'] button[class*='close']",
-    # Specific Gifi/Discount popups
     ".popin-close",
     "#popin-close",
     "a.close-popin",
-    # Centrakor Store Overlay
     "div.mod-shops button.close",
     ".mod-shops .close",
     "#shops-modal .close-button",
     ".ui-dialog-titlebar-close",
-    # Action
     "#cc-accept",
     ".cc-btn-accept",
     "button[data-testid='uc-accept-all-button']",

list_missing_sites.py

@@ -0,0 +1,62 @@
+"""
+Script to list all sites in database and check which ones are missing from search_config.py
+"""
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from app.database import SessionLocal
+from app.models import SearchSite
+from app.core.search_config import SITE_CONFIGS
+
+def main():
+    """List all sites in DB and identify missing configurations"""
+    db = SessionLocal()
+    
+    try:
+        # Get all sites from database
+        sites = db.query(SearchSite).order_by(SearchSite.name).all()
+        
+        print(f"\n{'='*80}")
+        print(f"Sites in Database: {len(sites)}")
+        print(f"Sites in SITE_CONFIGS: {len(SITE_CONFIGS)}")
+        print(f"{'='*80}\n")
+        
+        # Check each DB site
+        missing = []
+        configured = []
+        
+        for site in sites:
+            domain_clean = site.domain.replace("www.", "").lower()
+            
+            # Check if configured
+            is_configured = False
+            for key in SITE_CONFIGS.keys():
+                if key in domain_clean or domain_clean in key:
+                    configured.append((site.id, site.name, site.domain, key))
+                    is_configured = True
+                    break
+            
+            if not is_configured:
+                missing.append((site.id, site.name, site.domain))
+        
+        # Display results
+        if configured:
+            print("✅ CONFIGURED SITES:")
+            for sid, name, domain, key in configured:
+                print(f"   [{sid:2d}] {name:20s} ({domain:25s}) → {key}")
+        
+        if missing:
+            print(f"\n❌ MISSING {len(missing)} SITES:")
+            for sid, name, domain in missing:
+                print(f"   [{sid:2d}] {name:20s} ({domain})")
+        else:
+            print("\n✅ All sites are configured!")
+        
+        print(f"\n{'='*80}\n")
+        
+    finally:
+        db.close()
+
+if __name__ == "__main__":
+    main()