Merge pull request #24 from R0m1k3/claude/design-product-search-01Ea9dWZqZpuKFwBwVmp26PG

feat: Replace SearXNG with DuckDuckGo for product search

app/services/duckduckgo_service.py

@@ -0,0 +1,167 @@
+"""
+DuckDuckGo Search Service - Recherche de produits via DuckDuckGo
+"""
+
+import asyncio
+import logging
+from urllib.parse import urlparse
+
+from duckduckgo_search import DDGS
+
+logger = logging.getLogger(__name__)
+
+
+class SearchResult:
+    """Résultat de recherche DuckDuckGo"""
+
+    def __init__(self, url: str, title: str, snippet: str, source: str):
+        self.url = url
+        self.title = title
+        self.snippet = snippet
+        self.source = source
+
+    def __repr__(self):
+        return f"SearchResult(url={self.url}, title={self.title[:30]}...)"
+
+
+async def search(
+    query: str,
+    domains: list[str],
+    max_results: int = 20,
+    timeout: float = 15.0,
+) -> list[SearchResult]:
+    """
+    Effectue une recherche DuckDuckGo ciblée sur les domaines spécifiés.
+
+    Args:
+        query: Terme de recherche (ex: "lutin de noel")
+        domains: Liste des domaines à cibler (ex: ["amazon.fr", "fnac.com"])
+        max_results: Nombre maximum de résultats à retourner
+        timeout: Timeout en secondes
+
+    Returns:
+        Liste de SearchResult avec url, title, snippet, source
+    """
+    if not domains:
+        logger.warning("Aucun domaine spécifié pour la recherche")
+        return []
+
+    # Nettoyer les domaines (enlever protocole, www, etc.)
+    clean_domains = [_clean_domain(d) for d in domains]
+    clean_domains = [d for d in clean_domains if d]
+
+    # Construire la requête avec site: pour chaque domaine
+    site_query = " OR ".join([f"site:{domain}" for domain in clean_domains])
+    full_query = f"{query} ({site_query})"
+
+    logger.info(f"Recherche DuckDuckGo: {full_query}")
+
+    try:
+        # Exécuter la recherche dans un thread séparé (duckduckgo_search est synchrone)
+        loop = asyncio.get_event_loop()
+        results = await loop.run_in_executor(
+            None,
+            lambda: _do_search(full_query, max_results * 2, timeout),
+        )
+
+        # Filtrer pour ne garder que les domaines ciblés
+        filtered_results = []
+        seen_urls = set()
+
+        for item in results:
+            url = item.get("href", "") or item.get("link", "")
+            if not url:
+                continue
+
+            # Éviter les doublons
+            if url in seen_urls:
+                continue
+            seen_urls.add(url)
+
+            # Vérifier que l'URL appartient à un des domaines ciblés
+            url_domain = _extract_domain(url)
+            if not any(domain in url_domain for domain in clean_domains):
+                continue
+
+            filtered_results.append(
+                SearchResult(
+                    url=url,
+                    title=item.get("title", ""),
+                    snippet=item.get("body", "") or item.get("snippet", ""),
+                    source=url_domain,
+                )
+            )
+
+            if len(filtered_results) >= max_results:
+                break
+
+        logger.info(f"DuckDuckGo: {len(filtered_results)} résultats trouvés")
+        return filtered_results
+
+    except Exception as e:
+        logger.error(f"Erreur DuckDuckGo: {e}")
+        return []
+
+
+def _do_search(query: str, max_results: int, timeout: float) -> list[dict]:
+    """Effectue la recherche synchrone DuckDuckGo"""
+    try:
+        with DDGS() as ddgs:
+            results = list(ddgs.text(
+                query,
+                region="fr-fr",
+                safesearch="off",
+                max_results=max_results,
+            ))
+            return results
+    except Exception as e:
+        logger.error(f"Erreur lors de la recherche DuckDuckGo: {e}")
+        return []
+
+
+def _extract_domain(url: str) -> str:
+    """Extrait le domaine d'une URL"""
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        # Retirer www. si présent
+        if domain.startswith("www."):
+            domain = domain[4:]
+        return domain
+    except Exception:
+        return ""
+
+
+def _clean_domain(domain: str) -> str:
+    """Nettoie un domaine (enlève protocole, www, slash final, etc.)"""
+    if not domain:
+        return ""
+
+    domain = domain.strip()
+
+    # Si c'est une URL complète, extraire le domaine
+    if domain.startswith(("http://", "https://")):
+        domain = _extract_domain(domain)
+    else:
+        # Nettoyer le domaine directement
+        domain = domain.lower()
+        if domain.startswith("www."):
+            domain = domain[4:]
+        # Retirer le slash final
+        domain = domain.rstrip("/")
+
+    return domain
+
+
+async def health_check() -> bool:
+    """Vérifie que DuckDuckGo est accessible"""
+    try:
+        loop = asyncio.get_event_loop()
+        results = await loop.run_in_executor(
+            None,
+            lambda: _do_search("test", 1, 5.0),
+        )
+        return len(results) > 0
+    except Exception as e:
+        logger.warning(f"DuckDuckGo health check failed: {e}")
+        return False

app/services/search_service.py

@@ -1,6 +1,6 @@
 """
 Search Service - Orchestrateur de recherche de produits
-Combine SearXNG + Light Scraper + Browserless fallback
+Combine DuckDuckGo + Light Scraper + Browserless fallback
 """
 
 import asyncio
@@ -12,7 +12,7 @@ from sqlalchemy.orm import Session
 
 from app.models import SearchSite
 from app.schemas import SearchProgress, SearchResultItem
-from app.services import ai_service, light_scraper_service, searxng_service
+from app.services import ai_service, duckduckgo_service, light_scraper_service
 from app.services.scraper_service import ScraperService
 from app.services.settings_service import SettingsService
 
@@ -68,7 +68,7 @@ async def search_products(
     domains = [site.domain for site in sites]
     site_map = {site.domain: site for site in sites}
 
-    # Phase 1: Recherche SearXNG
+    # Phase 1: Recherche DuckDuckGo
     yield SearchProgress(
         status="searching",
         total=0,
@@ -77,13 +77,13 @@ async def search_products(
         results=[],
     )
 
-    searxng_results = await searxng_service.search(
+    search_results = await duckduckgo_service.search(
         query=query,
         domains=domains,
         max_results=max_results,
     )
 
-    if not searxng_results:
+    if not search_results:
         yield SearchProgress(
             status="error",
             total=0,
@@ -93,8 +93,8 @@ async def search_products(
         )
         return
 
-    total = len(searxng_results)
+    total = len(search_results)
-    logger.info(f"SearXNG: {total} URLs trouvées pour '{query}'")
+    logger.info(f"DuckDuckGo: {total} URLs trouvées pour '{query}'")
 
     # Phase 2: Scraping des URLs
     results: list[SearchResultItem] = []
@@ -103,10 +103,10 @@ async def search_products(
     # Créer un sémaphore pour limiter la concurrence
     semaphore = asyncio.Semaphore(parallel_limit)
 
-    async def process_url(searxng_result: searxng_service.SearXNGResult) -> SearchResultItem | None:
+    async def process_url(search_result: duckduckgo_service.SearchResult) -> SearchResultItem | None:
         async with semaphore:
-            url = searxng_result.url
+            url = search_result.url
-            domain = searxng_result.source
+            domain = search_result.source
             site = site_map.get(domain)
 
             if not site:
@@ -126,7 +126,7 @@ async def search_products(
                 if light_result.success and light_result.price is not None:
                     return SearchResultItem(
                         url=url,
-                        title=light_result.title or searxng_result.title,
+                        title=light_result.title or search_result.title,
                         price=light_result.price,
                         currency=light_result.currency,
                         in_stock=light_result.in_stock,
@@ -138,13 +138,13 @@ async def search_products(
 
             # Fallback: Browserless + IA
             try:
-                result = await _scrape_with_browserless(url, site_name, domain, searxng_result.title)
+                result = await _scrape_with_browserless(url, site_name, domain, search_result.title)
                 return result
             except Exception as e:
                 logger.error(f"Erreur scraping {url}: {e}")
                 return SearchResultItem(
                     url=url,
-                    title=searxng_result.title,
+                    title=search_result.title,
                     price=None,
                     site_name=site_name,
                     site_domain=domain,
@@ -152,7 +152,7 @@ async def search_products(
                 )
 
     # Traiter les URLs en parallèle avec mises à jour progressives
-    tasks = [asyncio.create_task(process_url(r)) for r in searxng_results]
+    tasks = [asyncio.create_task(process_url(r)) for r in search_results]
 
     for coro in asyncio.as_completed(tasks):
         try:

app/services/searxng_service.py

@@ -1,168 +0,0 @@
-"""
-SearXNG Service - Client pour le moteur de recherche auto-hébergé
-"""
-
-import logging
-import os
-from urllib.parse import quote_plus
-
-import httpx
-
-logger = logging.getLogger(__name__)
-
-SEARXNG_URL = os.getenv("SEARXNG_URL", "http://searxng:8080")
-
-
-class SearXNGResult:
-    """Résultat de recherche SearXNG"""
-
-    def __init__(self, url: str, title: str, snippet: str, source: str):
-        self.url = url
-        self.title = title
-        self.snippet = snippet
-        self.source = source
-
-    def __repr__(self):
-        return f"SearXNGResult(url={self.url}, title={self.title[:30]}...)"
-
-
-async def search(
-    query: str,
-    domains: list[str],
-    max_results: int = 20,
-    timeout: float = 15.0,
-) -> list[SearXNGResult]:
-    """
-    Effectue une recherche SearXNG ciblée sur les domaines spécifiés.
-
-    Args:
-        query: Terme de recherche (ex: "lutin de noel")
-        domains: Liste des domaines à cibler (ex: ["amazon.fr", "fnac.com"])
-        max_results: Nombre maximum de résultats à retourner
-        timeout: Timeout en secondes
-
-    Returns:
-        Liste de SearXNGResult avec url, title, snippet, source
-    """
-    if not domains:
-        logger.warning("Aucun domaine spécifié pour la recherche")
-        return []
-
-    # Nettoyer les domaines (enlever protocole, www, etc.)
-    clean_domains = [_clean_domain(d) for d in domains]
-    clean_domains = [d for d in clean_domains if d]  # Filtrer les vides
-
-    # Construire la requête avec site: pour chaque domaine
-    site_query = " OR ".join([f"site:{domain}" for domain in clean_domains])
-    full_query = f"{query} ({site_query})"
-
-    logger.info(f"Recherche SearXNG: {full_query}")
-
-    try:
-        async with httpx.AsyncClient(timeout=timeout) as client:
-            response = await client.get(
-                f"{SEARXNG_URL}/search",
-                params={
-                    "q": full_query,
-                    "format": "json",
-                    "language": "fr",
-                    "safesearch": 0,
-                    "pageno": 1,
-                },
-                headers={
-                    "Accept": "application/json",
-                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
-                    "X-Forwarded-For": "127.0.0.1",
-                    "X-Real-IP": "127.0.0.1",
-                },
-            )
-            response.raise_for_status()
-            data = response.json()
-
-            results = []
-            seen_urls = set()
-
-            for item in data.get("results", []):
-                url = item.get("url", "")
-
-                # Éviter les doublons
-                if url in seen_urls:
-                    continue
-                seen_urls.add(url)
-
-                # Vérifier que l'URL appartient à un des domaines ciblés
-                url_domain = _extract_domain(url)
-                if not any(domain in url_domain for domain in clean_domains):
-                    continue
-
-                results.append(
-                    SearXNGResult(
-                        url=url,
-                        title=item.get("title", ""),
-                        snippet=item.get("content", ""),
-                        source=url_domain,
-                    )
-                )
-
-                if len(results) >= max_results:
-                    break
-
-            logger.info(f"SearXNG: {len(results)} résultats trouvés")
-            return results
-
-    except httpx.TimeoutException:
-        logger.error(f"Timeout SearXNG après {timeout}s")
-        return []
-    except httpx.HTTPStatusError as e:
-        logger.error(f"Erreur HTTP SearXNG: {e.response.status_code}")
-        return []
-    except Exception as e:
-        logger.error(f"Erreur SearXNG: {e}")
-        return []
-
-
-def _extract_domain(url: str) -> str:
-    """Extrait le domaine d'une URL"""
-    try:
-        from urllib.parse import urlparse
-
-        parsed = urlparse(url)
-        domain = parsed.netloc.lower()
-        # Retirer www. si présent
-        if domain.startswith("www."):
-            domain = domain[4:]
-        return domain
-    except Exception:
-        return ""
-
-
-def _clean_domain(domain: str) -> str:
-    """Nettoie un domaine (enlève protocole, www, slash final, etc.)"""
-    if not domain:
-        return ""
-
-    domain = domain.strip()
-
-    # Si c'est une URL complète, extraire le domaine
-    if domain.startswith(("http://", "https://")):
-        domain = _extract_domain(domain)
-    else:
-        # Nettoyer le domaine directement
-        domain = domain.lower()
-        if domain.startswith("www."):
-            domain = domain[4:]
-        # Retirer le slash final
-        domain = domain.rstrip("/")
-
-    return domain
-
-
-async def health_check() -> bool:
-    """Vérifie que SearXNG est accessible"""
-    try:
-        async with httpx.AsyncClient(timeout=5.0) as client:
-            response = await client.get(f"{SEARXNG_URL}/healthz")
-            return response.status_code == 200
-    except Exception as e:
-        logger.warning(f"SearXNG health check failed: {e}")
-        return False

docker-compose.yml

@@ -12,8 +12,7 @@ services:
       - "8555:8555"
     environment:
       - DATABASE_URL=postgresql://priceflow:priceflow@db:5432/priceflow
-      - BROWSERLESS_URL=ws://browserless:3000
+      - BROWSERLESS_URL=ws://browserless:3012
-      - SEARXNG_URL=http://searxng:8080
       - LOG_LEVEL=INFO
       - CORS_ORIGINS=*
     depends_on:
@@ -21,8 +20,6 @@ services:
         condition: service_healthy
       browserless:
         condition: service_started
-      searxng:
-        condition: service_started
     volumes:
       - screenshots_data:/app/screenshots
     networks:
@@ -50,26 +47,6 @@ services:
     networks:
       - nginx_default
 
-  # SearXNG - Moteur de recherche auto-hébergé
-  searxng:
-    image: searxng/searxng:latest
-    container_name: priceflow-searxng
-    restart: unless-stopped
-    ports:
-      - "8888:8080"
-    environment:
-      - SEARXNG_BASE_URL=http://searxng:8080
-      - SEARXNG_SECRET=priceflow-searxng-secret
-    volumes:
-      # Mount entire config directory to avoid file/directory issues
-      - ./searxng:/etc/searxng:rw
-    dns:
-      - 8.8.8.8
-      - 8.8.4.4
-      - 1.1.1.1
-    networks:
-      - nginx_default
-
   # Browserless - Navigateur headless pour le scraping
   browserless:
     image: ghcr.io/browserless/chromium:latest

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "alembic",
     "tenacity",
     "slowapi",
+    "duckduckgo-search",
 ]
 
 [build-system]

searxng/limiter.toml

@@ -1,19 +0,0 @@
-# SearXNG rate limiter configuration
-# Disabled for internal use in PriceFlow
-
-[botdetection.ip_limit]
-# Disable IP rate limiting completely
-link_token = false
-
-[botdetection.ip_lists]
-# Allow all private network ranges + localhost
-pass_ip = [
-    "0.0.0.0/0",
-    "127.0.0.0/8",
-    "192.168.0.0/16",
-    "172.16.0.0/12",
-    "10.0.0.0/8",
-    "::/0",
-]
-# No blocked IPs
-block_ip = []

searxng/settings.yml

@@ -1,58 +0,0 @@
-# SearXNG settings for PriceFlow
-# Bot detection completely disabled for internal API use
-
-use_default_settings: true
-
-server:
-  secret_key: "priceflow-searxng-secret-key-change-in-production"
-  limiter: false
-  public_instance: false
-  image_proxy: false
-  port: 8080
-  bind_address: "0.0.0.0"
-  method: "GET"
-
-# Disable Redis (used for rate limiting)
-redis:
-  url: false
-
-search:
-  safe_search: 0
-  autocomplete: ""
-  default_lang: "fr"
-  formats:
-    - html
-    - json
-
-# Enable search engines
-engines:
-  - name: google
-    engine: google
-    shortcut: g
-    disabled: false
-
-  - name: bing
-    engine: bing
-    shortcut: b
-    disabled: false
-
-  - name: duckduckgo
-    engine: duckduckgo
-    shortcut: ddg
-    disabled: false
-
-  - name: qwant
-    engine: qwant
-    shortcut: qw
-    disabled: false
-
-outgoing:
-  request_timeout: 10.0
-  max_request_timeout: 15.0
-  useragent_suffix: ""
-  pool_connections: 100
-  pool_maxsize: 20
-
-general:
-  debug: false
-  instance_name: "PriceFlow Search"