From 39893e8c1ef8eaa0aae468ac43fa0c81e17a2f9f Mon Sep 17 00:00:00 2001
From: Michael SCHAL <michaelschal@gmail.com>
Date: Mon, 1 Dec 2025 01:00:25 +0100
Subject: [PATCH] feat: Add improved search service with persistent browser
 connection for e-commerce scraping.

---
 app/services/improved_search_service.py |  17 +-
 frontend/src/pages/Search.jsx           | 408 ++++++++++++------------
 2 files changed, 214 insertions(+), 211 deletions(-)

diff --git a/app/services/improved_search_service.py b/app/services/improved_search_service.py
index 8bb4f0d..5fe4bb3 100644
--- a/app/services/improved_search_service.py
+++ b/app/services/improved_search_service.py
@@ -1,6 +1,21 @@
 """
 Improved Search Service - Using Persistent Browser Connection
-from app.services.parsers import ParserFactory, ProductResult
+Based on ScraperService pattern for better session management and reliability
+"""
+
+import asyncio
+import json
+import logging
+import re
+from typing import AsyncGenerator, Optional
+from urllib.parse import quote_plus, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+from playwright.async_api import Browser, BrowserContext, Page, async_playwright
+
+from app.core.config import settings
+from app.core.search_config import SITE_CONFIGS, BROWSERLESS_URL
+from app.services.ai_price_extractor import AIPriceExtractor
 
 logger = logging.getLogger(__name__)
 
diff --git a/frontend/src/pages/Search.jsx b/frontend/src/pages/Search.jsx
index 68c4e26..64e5129 100644
--- a/frontend/src/pages/Search.jsx
+++ b/frontend/src/pages/Search.jsx
@@ -51,236 +51,224 @@ export default function Search() {
 
     // Charger les sites au démarrage
     useEffect(() => {
-        loadSites();
-    }, []);
+    } catch (error) {
+        console.error('Error loading sites:', error);
+        toast.error('Erreur lors du chargement des sites');
+    }
+};
 
-    const loadSites = async () => {
+const handleSearch = async (e) => {
+    e.preventDefault();
+    if (!query.trim() || !selectedSiteId) {
+        toast.error('Veuillez entrer une recherche et sélectionner un site');
+        return;
+    }
+
+    // Clear previous storage when starting new search
+    sessionStorage.removeItem(STORAGE_KEY);
+
+    // Fermer l'EventSource précédent si existant
+    if (eventSourceRef.current) {
+        eventSourceRef.current.close();
+    }
+
+    setIsSearching(true);
+    setResults([]);
+    setProgress({ status: 'searching', total: 1, completed: 0 });
+
+    // Construire l'URL avec les paramètres
+    const params = new URLSearchParams({
+        q: query.trim(),
+        sites: selectedSiteId,
+        max_results: 20,
+    });
+
+    // Créer l'EventSource pour SSE
+    const eventSource = new EventSource(`${API_URL}/search?${params}`);
+    eventSourceRef.current = eventSource;
+
+    eventSource.addEventListener('progress', (event) => {
         try {
-            const response = await axios.get(`${API_URL}/search-sites`);
-            const activeSites = response.data.filter(site => site.is_active);
-            setSites(activeSites);
-            // Sélectionner le premier site actif par défaut si aucun site n'est sélectionné
-            if (activeSites.length > 0 && !selectedSiteId) {
-                setSelectedSiteId(activeSites[0].id.toString());
-            }
-        } catch (error) {
-            console.error('Error loading sites:', error);
-            toast.error('Erreur lors du chargement des sites');
-        }
-    };
+            const data = JSON.parse(event.data);
+            setProgress(data);
+            setResults(data.results || []);
 
-    const handleSearch = async (e) => {
-        e.preventDefault();
-        if (!query.trim() || !selectedSiteId) {
-            toast.error('Veuillez entrer une recherche et sélectionner un site');
-            return;
-        }
+            if (data.status === 'completed' || data.status === 'error') {
+                setIsSearching(false);
+                eventSource.close();
 
-        // Clear previous storage when starting new search
-        sessionStorage.removeItem(STORAGE_KEY);
-
-        // Fermer l'EventSource précédent si existant
-        if (eventSourceRef.current) {
-            eventSourceRef.current.close();
-        }
-
-        setIsSearching(true);
-        setResults([]);
-        setProgress({ status: 'searching', total: 1, completed: 0 });
-
-        // Construire l'URL avec les paramètres
-        const params = new URLSearchParams({
-            q: query.trim(),
-            sites: selectedSiteId,
-            max_results: 20,
-        });
-
-        // Créer l'EventSource pour SSE
-        const eventSource = new EventSource(`${API_URL}/search?${params}`);
-        eventSourceRef.current = eventSource;
-
-        eventSource.addEventListener('progress', (event) => {
-            try {
-                const data = JSON.parse(event.data);
-                setProgress(data);
-                setResults(data.results || []);
-
-                if (data.status === 'completed' || data.status === 'error') {
-                    setIsSearching(false);
-                    eventSource.close();
-
-                    if (data.status === 'completed') {
-                        toast.success(`${data.results?.length || 0} produits trouvés`);
-                    } else if (data.message) {
-                        toast.error(data.message);
-                    }
+                if (data.status === 'completed') {
+                    toast.success(`${data.results?.length || 0} produits trouvés`);
+                } else if (data.message) {
+                    toast.error(data.message);
                 }
-            } catch (error) {
-                console.error('Error parsing SSE data:', error);
             }
-        });
-
-        eventSource.onerror = () => {
-            setIsSearching(false);
-            eventSource.close();
-            toast.error('Erreur de connexion au serveur');
-        };
-    };
-
-    const handleAddToMonitoring = async (result) => {
-        try {
-            await axios.post(`${API_URL}/items`, {
-                url: result.url,
-                name: result.title,
-                target_price: null,
-                check_interval_minutes: 60,
-            });
-            toast.success(`"${result.title}" ajouté au monitoring`);
         } catch (error) {
-            console.error('Error adding item:', error);
-            toast.error("Erreur lors de l'ajout au monitoring");
+            console.error('Error parsing SSE data:', error);
         }
+    });
+
+    eventSource.onerror = () => {
+        setIsSearching(false);
+        eventSource.close();
+        toast.error('Erreur de connexion au serveur');
     };
+};
 
-    const progressPercent = progress
-        ? Math.round((progress.completed / Math.max(progress.total, 1)) * 100)
-        : 0;
+const handleAddToMonitoring = async (result) => {
+    try {
+        await axios.post(`${API_URL}/items`, {
+            url: result.url,
+            name: result.title,
+            target_price: null,
+            check_interval_minutes: 60,
+        });
+        toast.success(`"${result.title}" ajouté au monitoring`);
+    } catch (error) {
+        console.error('Error adding item:', error);
+        toast.error("Erreur lors de l'ajout au monitoring");
+    }
+};
 
-    const selectedSite = sites.find(s => s.id.toString() === selectedSiteId);
+const progressPercent = progress
+    ? Math.round((progress.completed / Math.max(progress.total, 1)) * 100)
+    : 0;
 
-    return (
-        <div className="space-y-6 animate-in fade-in duration-500">
-            {/* Header */}
-            <div className="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
-                <div>
-                    <h2 className="text-2xl font-bold tracking-tight">Recherche de produits</h2>
-                    <p className="text-muted-foreground">
-                        Recherchez des produits sur un site et ajoutez-les au monitoring
-                    </p>
-                </div>
+const selectedSite = sites.find(s => s.id.toString() === selectedSiteId);
+
+return (
+    <div className="space-y-6 animate-in fade-in duration-500">
+        {/* Header */}
+        <div className="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
+            <div>
+                <h2 className="text-2xl font-bold tracking-tight">Recherche de produits</h2>
+                <p className="text-muted-foreground">
+                    Recherchez des produits sur un site et ajoutez-les au monitoring
+                </p>
             </div>
+        </div>
 
-            {/* Search Form */}
+        {/* Search Form */}
+        <Card>
+            <CardContent className="pt-6">
+                <form onSubmit={handleSearch} className="space-y-4">
+                    {/* Site Selection */}
+                    <div className="space-y-2">
+                        <p className="text-sm font-medium">Site de recherche :</p>
+                        {sites.length === 0 ? (
+                            <p className="text-sm text-muted-foreground">
+                                Aucun site configuré. Contactez l'administrateur pour activer des sites.
+                            </p>
+                        ) : (
+                            <Select
+                                value={selectedSiteId || ''}
+                                onValueChange={setSelectedSiteId}
+                                disabled={isSearching}
+                            >
+                                <SelectTrigger className="w-full md:w-[300px]">
+                                    <SelectValue placeholder="Sélectionner un site" />
+                                </SelectTrigger>
+                                <SelectContent>
+                                    {sites.map(site => (
+                                        <SelectItem key={site.id} value={site.id.toString()}>
+                                            <div className="flex items-center gap-2">
+                                                <span>{site.name}</span>
+                                                {site.category && (
+                                                    <span className="text-xs text-muted-foreground">({site.category})</span>
+                                                )}
+                                            </div>
+                                        </SelectItem>
+                                    ))}
+                                </SelectContent>
+                            </Select>
+                        )}
+                    </div>
+
+                    {/* Search Input */}
+                    <div className="flex gap-2">
+                        <div className="relative flex-1">
+                            <SearchIcon className="absolute left-3 top-3 h-4 w-4 text-muted-foreground" />
+                            <Input
+                                type="text"
+                                placeholder="Rechercher un produit..."
+                                value={query}
+                                onChange={(e) => setQuery(e.target.value)}
+                                className="pl-9"
+                                disabled={isSearching}
+                            />
+                        </div>
+                        <Button type="submit" disabled={isSearching || !query.trim() || !selectedSiteId}>
+                            {isSearching ? (
+                                <>
+                                    <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+                                    Recherche...
+                                </>
+                            ) : (
+                                <>
+                                    <SearchIcon className="h-4 w-4 mr-2" />
+                                    Rechercher
+                                </>
+                            )}
+                        </Button>
+                    </div>
+
+                    {/* Selected site info */}
+                    {selectedSite && (
+                        <p className="text-sm text-muted-foreground">
+                            Recherche sur <span className="font-medium">{selectedSite.name}</span> ({selectedSite.domain})
+                        </p>
+                    )}
+                </form>
+            </CardContent>
+        </Card>
+
+        {/* Progress Bar */}
+        {progress && isSearching && (
             <Card>
                 <CardContent className="pt-6">
-                    <form onSubmit={handleSearch} className="space-y-4">
-                        {/* Site Selection */}
-                        <div className="space-y-2">
-                            <p className="text-sm font-medium">Site de recherche :</p>
-                            {sites.length === 0 ? (
-                                <p className="text-sm text-muted-foreground">
-                                    Aucun site configuré. Contactez l'administrateur pour activer des sites.
-                                </p>
-                            ) : (
-                                <Select
-                                    value={selectedSiteId || ''}
-                                    onValueChange={setSelectedSiteId}
-                                    disabled={isSearching}
-                                >
-                                    <SelectTrigger className="w-full md:w-[300px]">
-                                        <SelectValue placeholder="Sélectionner un site" />
-                                    </SelectTrigger>
-                                    <SelectContent>
-                                        {sites.map(site => (
-                                            <SelectItem key={site.id} value={site.id.toString()}>
-                                                <div className="flex items-center gap-2">
-                                                    <span>{site.name}</span>
-                                                    {site.category && (
-                                                        <span className="text-xs text-muted-foreground">({site.category})</span>
-                                                    )}
-                                                </div>
-                                            </SelectItem>
-                                        ))}
-                                    </SelectContent>
-                                </Select>
-                            )}
+                    <div className="space-y-2">
+                        <div className="flex justify-between text-sm">
+                            <span>{progress.message || 'Recherche en cours...'}</span>
+                            <span>{progressPercent}%</span>
                         </div>
-
-                        {/* Search Input */}
-                        <div className="flex gap-2">
-                            <div className="relative flex-1">
-                                <SearchIcon className="absolute left-3 top-3 h-4 w-4 text-muted-foreground" />
-                                <Input
-                                    type="text"
-                                    placeholder="Rechercher un produit..."
-                                    value={query}
-                                    onChange={(e) => setQuery(e.target.value)}
-                                    className="pl-9"
-                                    disabled={isSearching}
-                                />
-                            </div>
-                            <Button type="submit" disabled={isSearching || !query.trim() || !selectedSiteId}>
-                                {isSearching ? (
-                                    <>
-                                        <Loader2 className="h-4 w-4 mr-2 animate-spin" />
-                                        Recherche...
-                                    </>
-                                ) : (
-                                    <>
-                                        <SearchIcon className="h-4 w-4 mr-2" />
-                                        Rechercher
-                                    </>
-                                )}
-                            </Button>
-                        </div>
-
-                        {/* Selected site info */}
-                        {selectedSite && (
-                            <p className="text-sm text-muted-foreground">
-                                Recherche sur <span className="font-medium">{selectedSite.name}</span> ({selectedSite.domain})
-                            </p>
-                        )}
-                    </form>
+                        <Progress value={progressPercent} />
+                    </div>
                 </CardContent>
             </Card>
+        )}
 
-            {/* Progress Bar */}
-            {progress && isSearching && (
-                <Card>
-                    <CardContent className="pt-6">
-                        <div className="space-y-2">
-                            <div className="flex justify-between text-sm">
-                                <span>{progress.message || 'Recherche en cours...'}</span>
-                                <span>{progressPercent}%</span>
-                            </div>
-                            <Progress value={progressPercent} />
-                        </div>
-                    </CardContent>
-                </Card>
-            )}
-
-            {/* Results */}
-            {results.length > 0 && (
-                <div className="space-y-4">
-                    <div className="flex items-center justify-between">
-                        <h3 className="text-lg font-semibold">
-                            {results.length} produit{results.length > 1 ? 's' : ''} trouvé{results.length > 1 ? 's' : ''}
-                        </h3>
-                    </div>
-                    <div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
-                        {results.map((result, index) => (
-                            <SearchResultCard
-                                key={`${result.url}-${index}`}
-                                result={result}
-                                onAddToMonitoring={() => handleAddToMonitoring(result)}
-                            />
-                        ))}
-                    </div>
+        {/* Results */}
+        {results.length > 0 && (
+            <div className="space-y-4">
+                <div className="flex items-center justify-between">
+                    <h3 className="text-lg font-semibold">
+                        {results.length} produit{results.length > 1 ? 's' : ''} trouvé{results.length > 1 ? 's' : ''}
+                    </h3>
                 </div>
-            )}
+                <div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
+                    {results.map((result, index) => (
+                        <SearchResultCard
+                            key={`${result.url}-${index}`}
+                            result={result}
+                            onAddToMonitoring={() => handleAddToMonitoring(result)}
+                        />
+                    ))}
+                </div>
+            </div>
+        )}
 
-            {/* Empty State */}
-            {!isSearching && results.length === 0 && progress?.status === 'completed' && (
-                <Card>
-                    <CardContent className="py-12 text-center">
-                        <SearchIcon className="h-12 w-12 mx-auto text-muted-foreground mb-4" />
-                        <h3 className="text-lg font-medium mb-2">Aucun produit trouvé</h3>
-                        <p className="text-muted-foreground">
-                            Essayez avec d'autres termes de recherche ou sélectionnez un autre site
-                        </p>
-                    </CardContent>
-                </Card>
-            )}
-        </div>
-    );
+        {/* Empty State */}
+        {!isSearching && results.length === 0 && progress?.status === 'completed' && (
+            <Card>
+                <CardContent className="py-12 text-center">
+                    <SearchIcon className="h-12 w-12 mx-auto text-muted-foreground mb-4" />
+                    <h3 className="text-lg font-medium mb-2">Aucun produit trouvé</h3>
+                    <p className="text-muted-foreground">
+                        Essayez avec d'autres termes de recherche ou sélectionnez un autre site
+                    </p>
+                </CardContent>
+            </Card>
+        )}
+    </div>
+);
 }