feat: Hardcode French e-commerce websites configuration

- Remove ability to add/edit/delete search sites from settings UI
- Configure 16 French e-commerce sites with search URLs:
  - Discount: Gifi, Stokomani, B&M, Centrakor, L'Incroyable, Action, La Foir'Fouille
  - Grandes Surfaces: E.Leclerc, Auchan, Carrefour
  - E-commerce: Amazon France, Cdiscount
  - Électronique: Darty, Boulanger, Fnac
- Add automatic database seeding on application startup
- Add /api/search-sites/seed and /api/search-sites/reset endpoints
- Add cookie consent banner handling for sites that require it
- Simplify Settings page to only show site list with toggle for active state
This commit is contained in:
Claude committed 2025-11-23 07:40:12 +00:00
1 parent c6a9ef681e
commit e1a9bd2657
5 files changed
+449 -468

No files matched your search

+234 -65
View File
@@ -39,99 +39,265 @@ class SearchResult:
# Configuration des sites avec leurs sélecteurs par défaut
# Liste des sites français configurés en dur
DEFAULT_SITE_CONFIGS = {
"amazon.fr": {
"search_url": "https://www.amazon.fr/s?k={query}",
"product_selector": "div[data-component-type='s-search-result'] h2 a",
"wait_selector": "div[data-component-type='s-search-result']",
},
"amazon.com": {
"search_url": "https://www.amazon.com/s?k={query}",
"product_selector": "div[data-component-type='s-search-result'] h2 a",
"wait_selector": "div[data-component-type='s-search-result']",
},
"action.com": {
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
"product_selector": "a.product-card__link, .product-card a, a[href*='/p/']",
"wait_selector": ".product-card, .search-results",
},
"fnac.com": {
"search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
"product_selector": "a.Article-title, .Article-item a.js-minifa-title",
"wait_selector": ".Article-item",
},
"cdiscount.com": {
"search_url": "https://www.cdiscount.com/search/10/{query}.html",
"product_selector": "a.prdtBImg, .prdtBILDetails a",
"wait_selector": ".prdtBILDetails",
},
"darty.com": {
"search_url": "https://www.darty.com/nav/recherche?text={query}",
"product_selector": ".product-card a, a[href*='/product']",
"wait_selector": ".product-card",
},
"boulanger.com": {
"search_url": "https://www.boulanger.com/resultats?tr={query}",
"product_selector": ".product-list__item a, a[href*='/ref/']",
"wait_selector": ".product-list",
},
# Magasins discount français
"stokomani.fr": {
"search_url": "https://www.stokomani.fr/recherche?q={query}",
"product_selector": (
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
".products article a, a[href*='/products/'], a[href*='/produit/']"
),
"wait_selector": ".products, .product-miniature, #js-product-list, .collection-products",
},
# === MAGASINS DISCOUNT ===
"gifi.fr": {
"name": "Gifi",
"search_url": "https://www.gifi.fr/catalogsearch/result/?q={query}",
"product_selector": (
".product-item a.product-item-link, .product-item-info a, "
".product-item a, .products-grid a, a[href$='.html']"
),
"wait_selector": ".products-grid, .product-items, .search-results, .products",
"category": "Discount",
"requires_js": True,
"priority": 1,
},
"stokomani.fr": {
"name": "Stokomani",
"search_url": "https://www.stokomani.fr/recherche?q={query}",
"product_selector": (
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
".products article a, a[href*='/products/'], a[href*='/produit/']"
),
"wait_selector": ".products, .product-miniature, #js-product-list, .collection-products",
"category": "Discount",
"requires_js": True,
"priority": 2,
},
"bmstores.fr": {
"name": "B&M",
"search_url": "https://bmstores.fr/module/ambjolisearch/jolisearch?s={query}",
"product_selector": (
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
".products article a, a[href*='/produits/']"
),
"wait_selector": ".products, .product-miniature, #js-product-list, #search_results",
"category": "Discount",
"requires_js": True,
"priority": 3,
},
"centrakor.com": {
"search_url": "https://www.centrakor.com/recherche?search={query}",
"product_selector": "a.product-card, .product-item a",
"wait_selector": ".product-card, .product-list",
"name": "Centrakor",
"search_url": "https://www.centrakor.com/catalogsearch/result/?q={query}",
"product_selector": (
".product-item a.product-item-link, .product-item-info a, "
".products-grid a, a[href$='.html']"
),
"wait_selector": ".products-grid, .product-items, .search-results",
"category": "Déco & Maison",
"requires_js": True,
"priority": 4,
},
"lincroyable.fr": {
"name": "L'Incroyable",
"search_url": "https://www.lincroyable.fr/recherche?q={query}",
"product_selector": (
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
".products article a, a[href*='/produit']"
),
"wait_selector": ".products, .product-miniature, #js-product-list",
"category": "Discount",
"requires_js": True,
"priority": 5,
},
"action.com": {
"name": "Action",
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
"product_selector": "a.product-card__link, .product-card a, a[href*='/p/']",
"wait_selector": ".product-card, .search-results",
"category": "Discount",
"requires_js": True,
"priority": 6,
},
"lafoirfouille.fr": {
"search_url": "https://www.lafoirfouille.fr/recherche?q={query}",
"product_selector": "a.product-item-link, .product-card a",
"wait_selector": ".product-item, .products",
"name": "La Foir'Fouille",
"search_url": "https://www.lafoirfouille.fr/catalogsearch/result/?q={query}",
"product_selector": (
".product-item a.product-item-link, .product-item-info a, "
".products-grid a, a[href$='.html']"
),
"wait_selector": ".products-grid, .product-items, .search-results",
"category": "Discount",
"requires_js": True,
"priority": 7,
},
"tedi.fr": {
"search_url": "https://www.tedi.fr/search?q={query}",
"product_selector": "a.product-card, .product a",
"wait_selector": ".product-card, .search-results",
},
"leclerc.fr": {
# === GRANDES SURFACES ===
"e.leclerc": {
"name": "E.Leclerc",
"search_url": "https://www.e.leclerc/recherche?q={query}",
"product_selector": "a.product-card, .product-tile a",
"wait_selector": ".product-card, .products-grid",
},
"carrefour.fr": {
"search_url": "https://www.carrefour.fr/s?q={query}",
"product_selector": "a.product-card, .product-item a",
"wait_selector": ".product-card, .product-list",
"product_selector": (
"a[data-testid='product-card-link'], .product-card a, "
".product-tile a, a[href*='/fp/']"
),
"wait_selector": ".product-card, .products-grid, [data-testid='product-card']",
"category": "Grande Surface",
"requires_js": True,
"priority": 8,
},
"auchan.fr": {
"name": "Auchan",
"search_url": "https://www.auchan.fr/recherche?text={query}",
"product_selector": "a.product-card, .product-tile a",
"wait_selector": ".product-card, .products",
"product_selector": (
".product-card a, .product-tile a, "
"a[href*='/p/'], .product-item a"
),
"wait_selector": ".product-card, .products, .product-list",
"category": "Grande Surface",
"requires_js": True,
"priority": 9,
},
"carrefour.fr": {
"name": "Carrefour",
"search_url": "https://www.carrefour.fr/s?q={query}",
"product_selector": (
"a[data-testid='product-card-link'], .product-card a, "
".product-item a, a[href*='/p/']"
),
"wait_selector": ".product-card, .product-list, [data-testid='product-card']",
"category": "Grande Surface",
"requires_js": True,
"priority": 10,
},
# === E-COMMERCE GÉNÉRALISTE ===
"amazon.fr": {
"name": "Amazon France",
"search_url": "https://www.amazon.fr/s?k={query}",
"product_selector": "div[data-component-type='s-search-result'] h2 a",
"wait_selector": "div[data-component-type='s-search-result']",
"category": "E-commerce",
"requires_js": True,
"priority": 11,
},
"cdiscount.com": {
"name": "Cdiscount",
"search_url": "https://www.cdiscount.com/search/10/{query}.html",
"product_selector": (
"a.prdtBImg, .prdtBILDetails a, .product-item a, "
"a[href*='/f-']"
),
"wait_selector": ".prdtBILDetails, .product-list",
"category": "E-commerce",
"requires_js": True,
"priority": 12,
},
# === ÉLECTRONIQUE / HIGH-TECH ===
"darty.com": {
"name": "Darty",
"search_url": "https://www.darty.com/nav/recherche?text={query}",
"product_selector": (
".product-card a, a[href*='/product'], "
".product-list-item a"
),
"wait_selector": ".product-card, .product-list",
"category": "Électronique",
"requires_js": True,
"priority": 13,
},
"boulanger.com": {
"name": "Boulanger",
"search_url": "https://www.boulanger.com/resultats?tr={query}",
"product_selector": (
".product-list__item a, a[href*='/ref/'], "
".product-card a"
),
"wait_selector": ".product-list, .product-card",
"category": "Électronique",
"requires_js": True,
"priority": 14,
},
# === AUTRES (conservés pour compatibilité) ===
"amazon.com": {
"name": "Amazon US",
"search_url": "https://www.amazon.com/s?k={query}",
"product_selector": "div[data-component-type='s-search-result'] h2 a",
"wait_selector": "div[data-component-type='s-search-result']",
"category": "E-commerce",
"requires_js": True,
"priority": 99,
},
"fnac.com": {
"name": "Fnac",
"search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
"product_selector": "a.Article-title, .Article-item a.js-minifa-title",
"wait_selector": ".Article-item",
"category": "Culture & Tech",
"requires_js": True,
"priority": 15,
},
}
# Sélecteurs communs pour les bannières de cookies
COOKIE_ACCEPT_SELECTORS = [
# Boutons d'acceptation génériques
"button[id*='accept']",
"button[id*='cookie']",
"button[class*='accept']",
"button[class*='cookie']",
"button[data-testid*='accept']",
"button[data-testid*='cookie']",
# Texte français
"button:has-text('Accepter')",
"button:has-text('J\\'accepte')",
"button:has-text('Tout accepter')",
"button:has-text('Accepter tout')",
"button:has-text('Accepter les cookies')",
"button:has-text('OK')",
"button:has-text('Continuer')",
# Texte anglais (certains sites utilisent l'anglais)
"button:has-text('Accept')",
"button:has-text('Accept all')",
"button:has-text('Accept cookies')",
"button:has-text('I accept')",
# Liens
"a:has-text('Accepter')",
"a:has-text('J\\'accepte')",
"a:has-text('Tout accepter')",
# Classes communes de CMP (Consent Management Platforms)
".didomi-continue-without-agreeing",
"#didomi-notice-agree-button",
".onetrust-accept-btn-handler",
"#onetrust-accept-btn-handler",
".tarteaucitronAllow",
"#tarteaucitronPersonalize",
".cc-btn.cc-allow",
".cky-btn-accept",
"#CybsAccept",
".qc-cmp2-summary-buttons button:first-child",
"[data-accept-cookies]",
"[data-gdpr-accept]",
]
async def _accept_cookies(page, domain: str) -> bool:
"""
Tente d'accepter les cookies sur une page.
Retourne True si un bouton a été cliqué, False sinon.
"""
try:
# Attendre un court moment pour que la bannière apparaisse
await asyncio.sleep(1)
for selector in COOKIE_ACCEPT_SELECTORS:
try:
button = page.locator(selector).first
if await button.is_visible(timeout=500):
await button.click(timeout=2000)
logger.info(f"{domain}: Cookies acceptés avec sélecteur {selector}")
# Attendre que la bannière disparaisse
await asyncio.sleep(0.5)
return True
except Exception:
continue
logger.debug(f"{domain}: Aucune bannière de cookies trouvée")
return False
except Exception as e:
logger.debug(f"{domain}: Erreur lors de l'acceptation des cookies: {e}")
return False
async def search(
query: str,
@@ -220,6 +386,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
# Naviguer vers la page de recherche
await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
# Accepter les cookies si nécessaire
await _accept_cookies(page, domain)
# Attendre que les résultats se chargent
if wait_selector:
try:
+93
View File
@@ -330,3 +330,96 @@ def delete_site(db: Session, site_id: int) -> bool:
db.delete(site)
db.commit()
return True
def seed_default_sites(db: Session) -> int:
"""
Initialise la base de données avec les sites configurés en dur.
Ne crée que les sites qui n'existent pas encore (basé sur le domaine).
Returns:
Nombre de sites créés
"""
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
created_count = 0
# Récupérer les domaines existants
existing_domains = {site.domain.lower().replace("www.", "")
for site in db.query(SearchSite).all()}
for domain, config in DEFAULT_SITE_CONFIGS.items():
# Normaliser le domaine pour comparaison
clean_domain = domain.lower().replace("www.", "")
# Ne pas créer si déjà existant
if clean_domain in existing_domains:
logger.debug(f"Site {domain} déjà existant, ignoré")
continue
# Créer le site
site_data = {
"name": config.get("name", domain),
"domain": domain,
"search_url": config.get("search_url"),
"product_link_selector": config.get("product_selector"),
"category": config.get("category"),
"requires_js": config.get("requires_js", True),
"priority": config.get("priority", 99),
"is_active": True,
}
try:
site = SearchSite(**site_data)
db.add(site)
db.commit()
created_count += 1
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
except Exception as e:
db.rollback()
logger.error(f"Erreur création site {domain}: {e}")
return created_count
def reset_sites_to_defaults(db: Session) -> int:
"""
Réinitialise tous les sites avec les valeurs par défaut.
Supprime tous les sites existants et recrée à partir de la config.
Returns:
Nombre de sites créés
"""
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
# Supprimer tous les sites existants
db.query(SearchSite).delete()
db.commit()
logger.info("Tous les sites supprimés")
# Recréer avec les valeurs par défaut
created_count = 0
for domain, config in DEFAULT_SITE_CONFIGS.items():
site_data = {
"name": config.get("name", domain),
"domain": domain,
"search_url": config.get("search_url"),
"product_link_selector": config.get("product_selector"),
"category": config.get("category"),
"requires_js": config.get("requires_js", True),
"priority": config.get("priority", 99),
"is_active": True,
}
try:
site = SearchSite(**site_data)
db.add(site)
db.commit()
created_count += 1
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
except Exception as e:
db.rollback()
logger.error(f"Erreur création site {domain}: {e}")
return created_count