diff --git a/app/core/search_config.py b/app/core/search_config.py index d234391..01bd94b 100644 --- a/app/core/search_config.py +++ b/app/core/search_config.py @@ -1,9 +1,5 @@ """ -Search Configuration -Centralizes all configuration for the search system: -- Site definitions (selectors, URLs) -- Proxy configurations -- User Agents +Search Configuration - COMPLETE FIXED VERSION """ import os @@ -14,7 +10,6 @@ BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") DEBUG_DUMPS_DIR = "/app/debug_dumps" # === PROXY CONFIGURATION === -# Format from user: ip:port:username:password AMAZON_PROXY_LIST_RAW = [ "142.111.48.253:7030:jasuwwjr:elbsx170nmnl", "31.59.20.176:6754:jasuwwjr:elbsx170nmnl", @@ -43,15 +38,11 @@ def get_amazon_proxies() -> list[dict]: # === USER AGENTS === USER_AGENT_POOL = [ - # Chrome Windows "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", - # Chrome Mac "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - # Firefox Windows "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", - # Recent Chrome "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", ] @@ -89,6 +80,15 @@ SITE_CONFIGS = { "category": "Discount", "requires_proxy": False, }, + "lafoirfouille.fr": { + "name": "La Foir'Fouille", + "search_url": "https://www.lafoirfouille.fr/recherche?controller=search&s={query}", + "product_selector": "article.product-miniature a.product-thumbnail, a.product_img_link", + "product_image_selector": "img[src], img[loading='lazy'], picture img", + "wait_selector": "article.product-miniature, .products", + "category": "Discount", + "requires_proxy": False, + }, "cdiscount.com": { "name": "Cdiscount", "search_url": "https://www.cdiscount.com/search/10/{query}.html", @@ -186,41 +186,37 @@ SITE_CONFIGS = { # === COOKIE BANNERS === COOKIE_ACCEPT_SELECTORS = [ - "#sp-cc-accept", # Amazon - "#onetrust-accept-btn-handler", # Many sites + "#sp-cc-accept", + "#onetrust-accept-btn-handler", ".onetrust-accept-btn-handler", - "#popin_tc_privacy_button_2", # Auchan/Cdiscount + "#popin_tc_privacy_button_2", ".popin_tc_privacy_button", - "#didomi-notice-agree-button", # Many French sites + "#didomi-notice-agree-button", ".didomi-continue-without-agreeing", - "[data-testid='accept-cookies-button']", # Carrefour - ".bcom-consent-accept-all", # Boulanger + "[data-testid='accept-cookies-button']", + ".bcom-consent-accept-all", "#cookieBanner-accept", - ".action-primary.action-accept", # Magento + ".action-primary.action-accept", "#btn-cookie-allow", "button:has-text('Tout accepter')", "button:has-text('Accepter tout')", "button:has-text('Accepter')", "button:has-text('J\\'accepte')", - # Promotional Popups / Modals ".modal-close", ".close-modal", ".popup-close", - "button[aria-label='Close']", + "button[aria-label='Close']", "button[aria-label='Fermer']", ".js-modal-close", "div[class*='popup'] button[class*='close']", "div[class*='modal'] button[class*='close']", - # Specific Gifi/Discount popups ".popin-close", "#popin-close", "a.close-popin", - # Centrakor Store Overlay "div.mod-shops button.close", ".mod-shops .close", "#shops-modal .close-button", ".ui-dialog-titlebar-close", - # Action "#cc-accept", ".cc-btn-accept", "button[data-testid='uc-accept-all-button']", diff --git a/list_missing_sites.py b/list_missing_sites.py new file mode 100644 index 0000000..6644217 --- /dev/null +++ b/list_missing_sites.py @@ -0,0 +1,62 @@ +""" +Script to list all sites in database and check which ones are missing from search_config.py +""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) + +from app.database import SessionLocal +from app.models import SearchSite +from app.core.search_config import SITE_CONFIGS + +def main(): + """List all sites in DB and identify missing configurations""" + db = SessionLocal() + + try: + # Get all sites from database + sites = db.query(SearchSite).order_by(SearchSite.name).all() + + print(f"\n{'='*80}") + print(f"Sites in Database: {len(sites)}") + print(f"Sites in SITE_CONFIGS: {len(SITE_CONFIGS)}") + print(f"{'='*80}\n") + + # Check each DB site + missing = [] + configured = [] + + for site in sites: + domain_clean = site.domain.replace("www.", "").lower() + + # Check if configured + is_configured = False + for key in SITE_CONFIGS.keys(): + if key in domain_clean or domain_clean in key: + configured.append((site.id, site.name, site.domain, key)) + is_configured = True + break + + if not is_configured: + missing.append((site.id, site.name, site.domain)) + + # Display results + if configured: + print("āœ… CONFIGURED SITES:") + for sid, name, domain, key in configured: + print(f" [{sid:2d}] {name:20s} ({domain:25s}) ā†’ {key}") + + if missing: + print(f"\nāŒ MISSING {len(missing)} SITES:") + for sid, name, domain in missing: + print(f" [{sid:2d}] {name:20s} ({domain})") + else: + print("\nāœ… All sites are configured!") + + print(f"\n{'='*80}\n") + + finally: + db.close() + +if __name__ == "__main__": + main()