diff --git a/app/services/duckduckgo_service.py b/app/services/duckduckgo_service.py new file mode 100644 index 0000000..8d5334e --- /dev/null +++ b/app/services/duckduckgo_service.py @@ -0,0 +1,167 @@ +""" +DuckDuckGo Search Service - Recherche de produits via DuckDuckGo +""" + +import asyncio +import logging +from urllib.parse import urlparse + +from duckduckgo_search import DDGS + +logger = logging.getLogger(__name__) + + +class SearchResult: + """Résultat de recherche DuckDuckGo""" + + def __init__(self, url: str, title: str, snippet: str, source: str): + self.url = url + self.title = title + self.snippet = snippet + self.source = source + + def __repr__(self): + return f"SearchResult(url={self.url}, title={self.title[:30]}...)" + + +async def search( + query: str, + domains: list[str], + max_results: int = 20, + timeout: float = 15.0, +) -> list[SearchResult]: + """ + Effectue une recherche DuckDuckGo ciblée sur les domaines spécifiés. + + Args: + query: Terme de recherche (ex: "lutin de noel") + domains: Liste des domaines à cibler (ex: ["amazon.fr", "fnac.com"]) + max_results: Nombre maximum de résultats à retourner + timeout: Timeout en secondes + + Returns: + Liste de SearchResult avec url, title, snippet, source + """ + if not domains: + logger.warning("Aucun domaine spécifié pour la recherche") + return [] + + # Nettoyer les domaines (enlever protocole, www, etc.) + clean_domains = [_clean_domain(d) for d in domains] + clean_domains = [d for d in clean_domains if d] + + # Construire la requête avec site: pour chaque domaine + site_query = " OR ".join([f"site:{domain}" for domain in clean_domains]) + full_query = f"{query} ({site_query})" + + logger.info(f"Recherche DuckDuckGo: {full_query}") + + try: + # Exécuter la recherche dans un thread séparé (duckduckgo_search est synchrone) + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + None, + lambda: _do_search(full_query, max_results * 2, timeout), + ) + + # Filtrer pour ne garder que les domaines ciblés + filtered_results = [] + seen_urls = set() + + for item in results: + url = item.get("href", "") or item.get("link", "") + if not url: + continue + + # Éviter les doublons + if url in seen_urls: + continue + seen_urls.add(url) + + # Vérifier que l'URL appartient à un des domaines ciblés + url_domain = _extract_domain(url) + if not any(domain in url_domain for domain in clean_domains): + continue + + filtered_results.append( + SearchResult( + url=url, + title=item.get("title", ""), + snippet=item.get("body", "") or item.get("snippet", ""), + source=url_domain, + ) + ) + + if len(filtered_results) >= max_results: + break + + logger.info(f"DuckDuckGo: {len(filtered_results)} résultats trouvés") + return filtered_results + + except Exception as e: + logger.error(f"Erreur DuckDuckGo: {e}") + return [] + + +def _do_search(query: str, max_results: int, timeout: float) -> list[dict]: + """Effectue la recherche synchrone DuckDuckGo""" + try: + with DDGS() as ddgs: + results = list(ddgs.text( + query, + region="fr-fr", + safesearch="off", + max_results=max_results, + )) + return results + except Exception as e: + logger.error(f"Erreur lors de la recherche DuckDuckGo: {e}") + return [] + + +def _extract_domain(url: str) -> str: + """Extrait le domaine d'une URL""" + try: + parsed = urlparse(url) + domain = parsed.netloc.lower() + # Retirer www. si présent + if domain.startswith("www."): + domain = domain[4:] + return domain + except Exception: + return "" + + +def _clean_domain(domain: str) -> str: + """Nettoie un domaine (enlève protocole, www, slash final, etc.)""" + if not domain: + return "" + + domain = domain.strip() + + # Si c'est une URL complète, extraire le domaine + if domain.startswith(("http://", "https://")): + domain = _extract_domain(domain) + else: + # Nettoyer le domaine directement + domain = domain.lower() + if domain.startswith("www."): + domain = domain[4:] + # Retirer le slash final + domain = domain.rstrip("/") + + return domain + + +async def health_check() -> bool: + """Vérifie que DuckDuckGo est accessible""" + try: + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + None, + lambda: _do_search("test", 1, 5.0), + ) + return len(results) > 0 + except Exception as e: + logger.warning(f"DuckDuckGo health check failed: {e}") + return False diff --git a/app/services/search_service.py b/app/services/search_service.py index 60ba385..009f56b 100644 --- a/app/services/search_service.py +++ b/app/services/search_service.py @@ -1,6 +1,6 @@ """ Search Service - Orchestrateur de recherche de produits -Combine SearXNG + Light Scraper + Browserless fallback +Combine DuckDuckGo + Light Scraper + Browserless fallback """ import asyncio @@ -12,7 +12,7 @@ from sqlalchemy.orm import Session from app.models import SearchSite from app.schemas import SearchProgress, SearchResultItem -from app.services import ai_service, light_scraper_service, searxng_service +from app.services import ai_service, duckduckgo_service, light_scraper_service from app.services.scraper_service import ScraperService from app.services.settings_service import SettingsService @@ -68,7 +68,7 @@ async def search_products( domains = [site.domain for site in sites] site_map = {site.domain: site for site in sites} - # Phase 1: Recherche SearXNG + # Phase 1: Recherche DuckDuckGo yield SearchProgress( status="searching", total=0, @@ -77,13 +77,13 @@ async def search_products( results=[], ) - searxng_results = await searxng_service.search( + search_results = await duckduckgo_service.search( query=query, domains=domains, max_results=max_results, ) - if not searxng_results: + if not search_results: yield SearchProgress( status="error", total=0, @@ -93,8 +93,8 @@ async def search_products( ) return - total = len(searxng_results) - logger.info(f"SearXNG: {total} URLs trouvées pour '{query}'") + total = len(search_results) + logger.info(f"DuckDuckGo: {total} URLs trouvées pour '{query}'") # Phase 2: Scraping des URLs results: list[SearchResultItem] = [] @@ -103,10 +103,10 @@ async def search_products( # Créer un sémaphore pour limiter la concurrence semaphore = asyncio.Semaphore(parallel_limit) - async def process_url(searxng_result: searxng_service.SearXNGResult) -> SearchResultItem | None: + async def process_url(search_result: duckduckgo_service.SearchResult) -> SearchResultItem | None: async with semaphore: - url = searxng_result.url - domain = searxng_result.source + url = search_result.url + domain = search_result.source site = site_map.get(domain) if not site: @@ -126,7 +126,7 @@ async def search_products( if light_result.success and light_result.price is not None: return SearchResultItem( url=url, - title=light_result.title or searxng_result.title, + title=light_result.title or search_result.title, price=light_result.price, currency=light_result.currency, in_stock=light_result.in_stock, @@ -138,13 +138,13 @@ async def search_products( # Fallback: Browserless + IA try: - result = await _scrape_with_browserless(url, site_name, domain, searxng_result.title) + result = await _scrape_with_browserless(url, site_name, domain, search_result.title) return result except Exception as e: logger.error(f"Erreur scraping {url}: {e}") return SearchResultItem( url=url, - title=searxng_result.title, + title=search_result.title, price=None, site_name=site_name, site_domain=domain, @@ -152,7 +152,7 @@ async def search_products( ) # Traiter les URLs en parallèle avec mises à jour progressives - tasks = [asyncio.create_task(process_url(r)) for r in searxng_results] + tasks = [asyncio.create_task(process_url(r)) for r in search_results] for coro in asyncio.as_completed(tasks): try: diff --git a/app/services/searxng_service.py b/app/services/searxng_service.py deleted file mode 100644 index f4924cb..0000000 --- a/app/services/searxng_service.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -SearXNG Service - Client pour le moteur de recherche auto-hébergé -""" - -import logging -import os -from urllib.parse import quote_plus - -import httpx - -logger = logging.getLogger(__name__) - -SEARXNG_URL = os.getenv("SEARXNG_URL", "http://searxng:8080") - - -class SearXNGResult: - """Résultat de recherche SearXNG""" - - def __init__(self, url: str, title: str, snippet: str, source: str): - self.url = url - self.title = title - self.snippet = snippet - self.source = source - - def __repr__(self): - return f"SearXNGResult(url={self.url}, title={self.title[:30]}...)" - - -async def search( - query: str, - domains: list[str], - max_results: int = 20, - timeout: float = 15.0, -) -> list[SearXNGResult]: - """ - Effectue une recherche SearXNG ciblée sur les domaines spécifiés. - - Args: - query: Terme de recherche (ex: "lutin de noel") - domains: Liste des domaines à cibler (ex: ["amazon.fr", "fnac.com"]) - max_results: Nombre maximum de résultats à retourner - timeout: Timeout en secondes - - Returns: - Liste de SearXNGResult avec url, title, snippet, source - """ - if not domains: - logger.warning("Aucun domaine spécifié pour la recherche") - return [] - - # Nettoyer les domaines (enlever protocole, www, etc.) - clean_domains = [_clean_domain(d) for d in domains] - clean_domains = [d for d in clean_domains if d] # Filtrer les vides - - # Construire la requête avec site: pour chaque domaine - site_query = " OR ".join([f"site:{domain}" for domain in clean_domains]) - full_query = f"{query} ({site_query})" - - logger.info(f"Recherche SearXNG: {full_query}") - - try: - async with httpx.AsyncClient(timeout=timeout) as client: - response = await client.get( - f"{SEARXNG_URL}/search", - params={ - "q": full_query, - "format": "json", - "language": "fr", - "safesearch": 0, - "pageno": 1, - }, - headers={ - "Accept": "application/json", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", - "X-Forwarded-For": "127.0.0.1", - "X-Real-IP": "127.0.0.1", - }, - ) - response.raise_for_status() - data = response.json() - - results = [] - seen_urls = set() - - for item in data.get("results", []): - url = item.get("url", "") - - # Éviter les doublons - if url in seen_urls: - continue - seen_urls.add(url) - - # Vérifier que l'URL appartient à un des domaines ciblés - url_domain = _extract_domain(url) - if not any(domain in url_domain for domain in clean_domains): - continue - - results.append( - SearXNGResult( - url=url, - title=item.get("title", ""), - snippet=item.get("content", ""), - source=url_domain, - ) - ) - - if len(results) >= max_results: - break - - logger.info(f"SearXNG: {len(results)} résultats trouvés") - return results - - except httpx.TimeoutException: - logger.error(f"Timeout SearXNG après {timeout}s") - return [] - except httpx.HTTPStatusError as e: - logger.error(f"Erreur HTTP SearXNG: {e.response.status_code}") - return [] - except Exception as e: - logger.error(f"Erreur SearXNG: {e}") - return [] - - -def _extract_domain(url: str) -> str: - """Extrait le domaine d'une URL""" - try: - from urllib.parse import urlparse - - parsed = urlparse(url) - domain = parsed.netloc.lower() - # Retirer www. si présent - if domain.startswith("www."): - domain = domain[4:] - return domain - except Exception: - return "" - - -def _clean_domain(domain: str) -> str: - """Nettoie un domaine (enlève protocole, www, slash final, etc.)""" - if not domain: - return "" - - domain = domain.strip() - - # Si c'est une URL complète, extraire le domaine - if domain.startswith(("http://", "https://")): - domain = _extract_domain(domain) - else: - # Nettoyer le domaine directement - domain = domain.lower() - if domain.startswith("www."): - domain = domain[4:] - # Retirer le slash final - domain = domain.rstrip("/") - - return domain - - -async def health_check() -> bool: - """Vérifie que SearXNG est accessible""" - try: - async with httpx.AsyncClient(timeout=5.0) as client: - response = await client.get(f"{SEARXNG_URL}/healthz") - return response.status_code == 200 - except Exception as e: - logger.warning(f"SearXNG health check failed: {e}") - return False diff --git a/docker-compose.yml b/docker-compose.yml index 85f45fc..3e2b399 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,7 @@ services: - "8555:8555" environment: - DATABASE_URL=postgresql://priceflow:priceflow@db:5432/priceflow - - BROWSERLESS_URL=ws://browserless:3000 - - SEARXNG_URL=http://searxng:8080 + - BROWSERLESS_URL=ws://browserless:3012 - LOG_LEVEL=INFO - CORS_ORIGINS=* depends_on: @@ -21,8 +20,6 @@ services: condition: service_healthy browserless: condition: service_started - searxng: - condition: service_started volumes: - screenshots_data:/app/screenshots networks: @@ -50,26 +47,6 @@ services: networks: - nginx_default - # SearXNG - Moteur de recherche auto-hébergé - searxng: - image: searxng/searxng:latest - container_name: priceflow-searxng - restart: unless-stopped - ports: - - "8888:8080" - environment: - - SEARXNG_BASE_URL=http://searxng:8080 - - SEARXNG_SECRET=priceflow-searxng-secret - volumes: - # Mount entire config directory to avoid file/directory issues - - ./searxng:/etc/searxng:rw - dns: - - 8.8.8.8 - - 8.8.4.4 - - 1.1.1.1 - networks: - - nginx_default - # Browserless - Navigateur headless pour le scraping browserless: image: ghcr.io/browserless/chromium:latest diff --git a/pyproject.toml b/pyproject.toml index 713ab33..e7eb9b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "alembic", "tenacity", "slowapi", + "duckduckgo-search", ] [build-system] diff --git a/searxng/limiter.toml b/searxng/limiter.toml deleted file mode 100644 index 754c10f..0000000 --- a/searxng/limiter.toml +++ /dev/null @@ -1,19 +0,0 @@ -# SearXNG rate limiter configuration -# Disabled for internal use in PriceFlow - -[botdetection.ip_limit] -# Disable IP rate limiting completely -link_token = false - -[botdetection.ip_lists] -# Allow all private network ranges + localhost -pass_ip = [ - "0.0.0.0/0", - "127.0.0.0/8", - "192.168.0.0/16", - "172.16.0.0/12", - "10.0.0.0/8", - "::/0", -] -# No blocked IPs -block_ip = [] diff --git a/searxng/settings.yml b/searxng/settings.yml deleted file mode 100644 index 8ba7f2e..0000000 --- a/searxng/settings.yml +++ /dev/null @@ -1,58 +0,0 @@ -# SearXNG settings for PriceFlow -# Bot detection completely disabled for internal API use - -use_default_settings: true - -server: - secret_key: "priceflow-searxng-secret-key-change-in-production" - limiter: false - public_instance: false - image_proxy: false - port: 8080 - bind_address: "0.0.0.0" - method: "GET" - -# Disable Redis (used for rate limiting) -redis: - url: false - -search: - safe_search: 0 - autocomplete: "" - default_lang: "fr" - formats: - - html - - json - -# Enable search engines -engines: - - name: google - engine: google - shortcut: g - disabled: false - - - name: bing - engine: bing - shortcut: b - disabled: false - - - name: duckduckgo - engine: duckduckgo - shortcut: ddg - disabled: false - - - name: qwant - engine: qwant - shortcut: qw - disabled: false - -outgoing: - request_timeout: 10.0 - max_request_timeout: 15.0 - useragent_suffix: "" - pool_connections: 100 - pool_maxsize: 20 - -general: - debug: false - instance_name: "PriceFlow Search"