From bcc3b0900ff7d90587ce503beebe916aeece9f23 Mon Sep 17 00:00:00 2001
From: Michael SCHAL <michaelschal@gmail.com>
Date: Sat, 29 Nov 2025 10:32:01 +0100
Subject: [PATCH] feat: Add core search configuration and service for web
 scraping, including site-specific selectors, proxies, and user agents.

---
 app/core/search_config.py      | 41 +++++++++-------------------------
 app/services/search_service.py | 16 +++++++++----
 2 files changed, 22 insertions(+), 35 deletions(-)

diff --git a/app/core/search_config.py b/app/core/search_config.py
index f041384..f2c9656 100644
--- a/app/core/search_config.py
+++ b/app/core/search_config.py
@@ -70,40 +70,14 @@ SITE_CONFIGS = {
     },
     "stokomani.fr": {
         "name": "Stokomani",
-        "search_url": "https://www.stokomani.fr/recherche?q={query}",
-        "product_selector": ".product-miniature a, .js-product-miniature a, a.product-thumbnail",
-        "wait_selector": ".products, .product-miniature",
+        "search_url": "https://www.stokomani.fr/search?options%5Bprefix%5D=last&q={query}",
+        "product_selector": "a[href^='/products/']",
+        "wait_selector": "a[href^='/products/']",
         "category": "Discount",
         "requires_proxy": False,
     },
     "bmstores.fr": {
         "name": "B&M",
-        "search_url": "https://bmstores.fr/module/ambjolisearch/jolisearch?s={query}",
-        "product_selector": "a.product-thumbnail[href*='/produits/']:not([href*='/c/'])",
-        "wait_selector": ".products, .product-miniature",
-        "category": "Discount",
-        "requires_proxy": False,
-    },
-    "centrakor.com": {
-        "name": "Centrakor",
-        "search_url": "https://www.centrakor.com/catalogsearch/result/?q={query}",
-        "product_selector": ".product-item a.product-item-link, .product-item-info a",
-        "wait_selector": ".products-grid, .product-items",
-        "category": "Déco & Maison",
-        "requires_proxy": False,
-    },
-    "lincroyable.fr": {
-        "name": "L'Incroyable",
-        "search_url": "https://www.lincroyable.fr/recherche-query={query}/?",
-        "product_selector": ".product-miniature a, .js-product-miniature a",
-        "wait_selector": ".products, .product-miniature",
-        "category": "Discount",
-        "requires_proxy": False,
-    },
-    "action.com": {
-        "name": "Action",
-        "search_url": "https://www.action.com/fr-fr/search/?q={query}",
-        "product_selector": "a.product-card__link, .product-card a",
         "wait_selector": ".product-card, .search-results",
         "category": "Discount",
         "requires_proxy": False,
@@ -136,8 +110,8 @@ SITE_CONFIGS = {
     "carrefour.fr": {
         "name": "Carrefour",
         "search_url": "https://www.carrefour.fr/s?q={query}",
-        "product_selector": "a[href*='/p/'][href$='.html']",
-        "wait_selector": ".search-results, .product-grid",
+        "product_selector": "a.product-card-click-wrapper[href^='/p/']",
+        "wait_selector": "a.product-card-click-wrapper[href^='/p/']",
         "category": "Grande Surface",
         "requires_proxy": False,
     },
@@ -216,4 +190,9 @@ COOKIE_ACCEPT_SELECTORS = [
     ".popin-close",
     "#popin-close",
     "a.close-popin",
+    # Centrakor Store Overlay
+    "div.mod-shops button.close",
+    ".mod-shops .close",
+    "#shops-modal .close-button",
+    ".ui-dialog-titlebar-close",
 ]
diff --git a/app/services/search_service.py b/app/services/search_service.py
index 6f2b01f..ecaf7f1 100644
--- a/app/services/search_service.py
+++ b/app/services/search_service.py
@@ -234,10 +234,18 @@ class NewSearchService:
         # Phase 1: Parse results
         initial_results = NewSearchService._parse_results(html_content, site_key, search_url)
         
-        # Phase 2: Scrape details for each result (Sequential or Limited Parallel)
-        # We yield as soon as one is done
-        for res in initial_results:
-            enriched_res = await NewSearchService.scrape_item(res)
+        # Phase 2: Scrape details for each result (Parallel)
+        # We want to yield results as they complete, not wait for all
+        semaphore = asyncio.Semaphore(3) # Limit concurrency per site
+
+        async def scrape_wrapper(res):
+            async with semaphore:
+                return await NewSearchService.scrape_item(res)
+
+        tasks = [scrape_wrapper(r) for r in initial_results]
+        
+        for future in asyncio.as_completed(tasks):
+            enriched_res = await future
             yield enriched_res
 
     @staticmethod