mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-23 23:28:12 +02:00
feat: Hardcode French e-commerce websites configuration
- Remove ability to add/edit/delete search sites from settings UI - Configure 16 French e-commerce sites with search URLs: - Discount: Gifi, Stokomani, B&M, Centrakor, L'Incroyable, Action, La Foir'Fouille - Grandes Surfaces: E.Leclerc, Auchan, Carrefour - E-commerce: Amazon France, Cdiscount - Électronique: Darty, Boulanger, Fnac - Add automatic database seeding on application startup - Add /api/search-sites/seed and /api/search-sites/reset endpoints - Add cookie consent banner handling for sites that require it - Simplify Settings page to only show site list with toggle for active state
This commit is contained in:
5 files changed
+449
-468
No files matched your search
+17
-1
@@ -11,10 +11,11 @@ from slowapi import _rate_limit_exceeded_handler
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.database import engine
|
||||
from app.database import SessionLocal, engine
|
||||
from app.limiter import limiter
|
||||
from app.routers import items, jobs, notifications, openrouter, search, search_sites, settings
|
||||
from app.services.scheduler_service import scheduled_refresh, scheduler
|
||||
from app.services import search_service
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
@@ -80,6 +81,21 @@ async def lifespan(app: FastAPI):
|
||||
except Exception as e:
|
||||
logger.error(f"Migration error: {e}")
|
||||
|
||||
# Seed default sites
|
||||
logger.info("Seeding default search sites...")
|
||||
try:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
created = search_service.seed_default_sites(db)
|
||||
if created > 0:
|
||||
logger.info(f"{created} site(s) de recherche créé(s)")
|
||||
else:
|
||||
logger.info("Sites de recherche déjà initialisés")
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Error seeding sites: {e}")
|
||||
|
||||
logger.info("Starting smart scheduler (Heartbeat: 1 minute)")
|
||||
scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True)
|
||||
scheduler.start()
|
||||
|
||||
+44
-142
@@ -1,17 +1,16 @@
|
||||
"""
|
||||
API Router pour la gestion des sites de recherche
|
||||
Les sites sont configurés en dur dans direct_search_service.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal, get_db
|
||||
from app.schemas import SearchSiteCreate, SearchSiteResponse, SearchSiteUpdate
|
||||
from app.database import get_db
|
||||
from app.schemas import SearchSiteResponse, SearchSiteUpdate
|
||||
from app.services import search_service
|
||||
from app.services.search_url_discovery import discover_search_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,157 +33,60 @@ async def get_site(site_id: int, db: Session = Depends(get_db)):
|
||||
return site
|
||||
|
||||
|
||||
async def _discover_and_update_site(site_id: int, domain: str, db_url: str):
|
||||
"""Découvre l'URL de recherche en background et met à jour le site"""
|
||||
try:
|
||||
logger.info(f"Discovering search URL for site {site_id} ({domain})")
|
||||
result = await discover_search_url(domain)
|
||||
|
||||
if result.get("discovered"):
|
||||
# Mettre à jour le site avec l'URL découverte
|
||||
db = SessionLocal()
|
||||
try:
|
||||
update_data = {}
|
||||
if result.get("search_url"):
|
||||
update_data["search_url"] = result["search_url"]
|
||||
if result.get("product_link_selector"):
|
||||
update_data["product_link_selector"] = result["product_link_selector"]
|
||||
|
||||
if update_data:
|
||||
search_service.update_site(db, site_id, update_data)
|
||||
logger.info(f"Updated site {site_id} with discovered URL: {result.get('search_url')}")
|
||||
finally:
|
||||
db.close()
|
||||
else:
|
||||
logger.warning(f"Could not discover search URL for {domain}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in background discovery for {domain}: {e}")
|
||||
|
||||
|
||||
@router.post("", response_model=SearchSiteResponse, status_code=201)
|
||||
async def create_site(
|
||||
site: SearchSiteCreate,
|
||||
background_tasks: BackgroundTasks,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Crée un nouveau site de recherche.
|
||||
Si search_url n'est pas fourni, tente de le découvrir automatiquement.
|
||||
"""
|
||||
try:
|
||||
new_site = search_service.create_site(db, site.model_dump())
|
||||
|
||||
# Si pas de search_url, lancer la découverte en background
|
||||
if not site.search_url:
|
||||
db_url = os.getenv("DATABASE_URL", "")
|
||||
background_tasks.add_task(
|
||||
_discover_and_update_site,
|
||||
new_site.id,
|
||||
new_site.domain,
|
||||
db_url,
|
||||
)
|
||||
logger.info(f"Started background discovery for site {new_site.id}")
|
||||
|
||||
return new_site
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
|
||||
|
||||
@router.put("/{site_id}", response_model=SearchSiteResponse)
|
||||
async def update_site(
|
||||
site_id: int,
|
||||
site: SearchSiteUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Met à jour un site de recherche"""
|
||||
updated = search_service.update_site(
|
||||
db, site_id, site.model_dump(exclude_unset=True)
|
||||
)
|
||||
"""
|
||||
Met à jour un site de recherche.
|
||||
Permet uniquement de modifier is_active et priority.
|
||||
"""
|
||||
# Limiter les champs modifiables
|
||||
allowed_fields = {"is_active", "priority"}
|
||||
update_data = site.model_dump(exclude_unset=True)
|
||||
filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields}
|
||||
|
||||
if not filtered_data:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés"
|
||||
)
|
||||
|
||||
updated = search_service.update_site(db, site_id, filtered_data)
|
||||
if not updated:
|
||||
raise HTTPException(status_code=404, detail="Site not found")
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/{site_id}", status_code=204)
|
||||
async def delete_site(site_id: int, db: Session = Depends(get_db)):
|
||||
"""Supprime un site de recherche"""
|
||||
deleted = search_service.delete_site(db, site_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="Site not found")
|
||||
|
||||
|
||||
@router.post("/{site_id}/discover", response_model=dict)
|
||||
async def discover_site_search_url(
|
||||
site_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
@router.post("/seed", response_model=dict)
|
||||
async def seed_default_sites(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Force la découverte de l'URL de recherche pour un site existant.
|
||||
Utile si la découverte automatique n'a pas fonctionné.
|
||||
Initialise les sites par défaut s'ils n'existent pas encore.
|
||||
Utile après une première installation.
|
||||
"""
|
||||
site = search_service.get_site_by_id(db, site_id)
|
||||
if not site:
|
||||
raise HTTPException(status_code=404, detail="Site not found")
|
||||
|
||||
result = await discover_search_url(site.domain)
|
||||
|
||||
if result.get("discovered"):
|
||||
# Mettre à jour le site
|
||||
update_data = {}
|
||||
if result.get("search_url"):
|
||||
update_data["search_url"] = result["search_url"]
|
||||
if result.get("product_link_selector"):
|
||||
update_data["product_link_selector"] = result["product_link_selector"]
|
||||
|
||||
if update_data:
|
||||
search_service.update_site(db, site_id, update_data)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"URL de recherche découverte: {result.get('search_url')}",
|
||||
"search_url": result.get("search_url"),
|
||||
"product_link_selector": result.get("product_link_selector"),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"message": "Impossible de découvrir l'URL de recherche automatiquement",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/discover-all", response_model=dict)
|
||||
async def discover_all_missing_search_urls(
|
||||
background_tasks: BackgroundTasks,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Lance la découverte d'URL de recherche pour tous les sites qui n'en ont pas.
|
||||
La découverte se fait en arrière-plan.
|
||||
"""
|
||||
sites = search_service.get_all_sites(db)
|
||||
sites_to_discover = [s for s in sites if not s.search_url]
|
||||
|
||||
if not sites_to_discover:
|
||||
return {
|
||||
"success": True,
|
||||
"message": "Tous les sites ont déjà une URL de recherche configurée",
|
||||
"sites_queued": 0,
|
||||
}
|
||||
|
||||
db_url = os.getenv("DATABASE_URL", "")
|
||||
|
||||
for site in sites_to_discover:
|
||||
background_tasks.add_task(
|
||||
_discover_and_update_site,
|
||||
site.id,
|
||||
site.domain,
|
||||
db_url,
|
||||
)
|
||||
created = search_service.seed_default_sites(db)
|
||||
total = len(search_service.get_all_sites(db))
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Découverte lancée pour {len(sites_to_discover)} sites",
|
||||
"sites_queued": len(sites_to_discover),
|
||||
"sites": [{"id": s.id, "domain": s.domain} for s in sites_to_discover],
|
||||
"message": f"{created} nouveau(x) site(s) créé(s)",
|
||||
"sites_created": created,
|
||||
"total_sites": total,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/reset", response_model=dict)
|
||||
async def reset_sites_to_defaults(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Réinitialise tous les sites avec les valeurs par défaut.
|
||||
ATTENTION: Supprime tous les sites existants et recrée à partir de la config.
|
||||
"""
|
||||
created = search_service.reset_sites_to_defaults(db)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Sites réinitialisés. {created} site(s) créé(s)",
|
||||
"sites_created": created,
|
||||
}
|
||||
@@ -39,99 +39,265 @@ class SearchResult:
|
||||
|
||||
|
||||
# Configuration des sites avec leurs sélecteurs par défaut
|
||||
# Liste des sites français configurés en dur
|
||||
DEFAULT_SITE_CONFIGS = {
|
||||
"amazon.fr": {
|
||||
"search_url": "https://www.amazon.fr/s?k={query}",
|
||||
"product_selector": "div[data-component-type='s-search-result'] h2 a",
|
||||
"wait_selector": "div[data-component-type='s-search-result']",
|
||||
},
|
||||
"amazon.com": {
|
||||
"search_url": "https://www.amazon.com/s?k={query}",
|
||||
"product_selector": "div[data-component-type='s-search-result'] h2 a",
|
||||
"wait_selector": "div[data-component-type='s-search-result']",
|
||||
},
|
||||
"action.com": {
|
||||
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
||||
"product_selector": "a.product-card__link, .product-card a, a[href*='/p/']",
|
||||
"wait_selector": ".product-card, .search-results",
|
||||
},
|
||||
"fnac.com": {
|
||||
"search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
|
||||
"product_selector": "a.Article-title, .Article-item a.js-minifa-title",
|
||||
"wait_selector": ".Article-item",
|
||||
},
|
||||
"cdiscount.com": {
|
||||
"search_url": "https://www.cdiscount.com/search/10/{query}.html",
|
||||
"product_selector": "a.prdtBImg, .prdtBILDetails a",
|
||||
"wait_selector": ".prdtBILDetails",
|
||||
},
|
||||
"darty.com": {
|
||||
"search_url": "https://www.darty.com/nav/recherche?text={query}",
|
||||
"product_selector": ".product-card a, a[href*='/product']",
|
||||
"wait_selector": ".product-card",
|
||||
},
|
||||
"boulanger.com": {
|
||||
"search_url": "https://www.boulanger.com/resultats?tr={query}",
|
||||
"product_selector": ".product-list__item a, a[href*='/ref/']",
|
||||
"wait_selector": ".product-list",
|
||||
},
|
||||
# Magasins discount français
|
||||
"stokomani.fr": {
|
||||
"search_url": "https://www.stokomani.fr/recherche?q={query}",
|
||||
"product_selector": (
|
||||
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
|
||||
".products article a, a[href*='/products/'], a[href*='/produit/']"
|
||||
),
|
||||
"wait_selector": ".products, .product-miniature, #js-product-list, .collection-products",
|
||||
},
|
||||
# === MAGASINS DISCOUNT ===
|
||||
"gifi.fr": {
|
||||
"name": "Gifi",
|
||||
"search_url": "https://www.gifi.fr/catalogsearch/result/?q={query}",
|
||||
"product_selector": (
|
||||
".product-item a.product-item-link, .product-item-info a, "
|
||||
".product-item a, .products-grid a, a[href$='.html']"
|
||||
),
|
||||
"wait_selector": ".products-grid, .product-items, .search-results, .products",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 1,
|
||||
},
|
||||
"stokomani.fr": {
|
||||
"name": "Stokomani",
|
||||
"search_url": "https://www.stokomani.fr/recherche?q={query}",
|
||||
"product_selector": (
|
||||
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
|
||||
".products article a, a[href*='/products/'], a[href*='/produit/']"
|
||||
),
|
||||
"wait_selector": ".products, .product-miniature, #js-product-list, .collection-products",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 2,
|
||||
},
|
||||
"bmstores.fr": {
|
||||
"name": "B&M",
|
||||
"search_url": "https://bmstores.fr/module/ambjolisearch/jolisearch?s={query}",
|
||||
"product_selector": (
|
||||
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
|
||||
".products article a, a[href*='/produits/']"
|
||||
),
|
||||
"wait_selector": ".products, .product-miniature, #js-product-list, #search_results",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 3,
|
||||
},
|
||||
"centrakor.com": {
|
||||
"search_url": "https://www.centrakor.com/recherche?search={query}",
|
||||
"product_selector": "a.product-card, .product-item a",
|
||||
"wait_selector": ".product-card, .product-list",
|
||||
"name": "Centrakor",
|
||||
"search_url": "https://www.centrakor.com/catalogsearch/result/?q={query}",
|
||||
"product_selector": (
|
||||
".product-item a.product-item-link, .product-item-info a, "
|
||||
".products-grid a, a[href$='.html']"
|
||||
),
|
||||
"wait_selector": ".products-grid, .product-items, .search-results",
|
||||
"category": "Déco & Maison",
|
||||
"requires_js": True,
|
||||
"priority": 4,
|
||||
},
|
||||
"lincroyable.fr": {
|
||||
"name": "L'Incroyable",
|
||||
"search_url": "https://www.lincroyable.fr/recherche?q={query}",
|
||||
"product_selector": (
|
||||
".product-miniature a, .js-product-miniature a, a.product-thumbnail, "
|
||||
".products article a, a[href*='/produit']"
|
||||
),
|
||||
"wait_selector": ".products, .product-miniature, #js-product-list",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 5,
|
||||
},
|
||||
"action.com": {
|
||||
"name": "Action",
|
||||
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
||||
"product_selector": "a.product-card__link, .product-card a, a[href*='/p/']",
|
||||
"wait_selector": ".product-card, .search-results",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 6,
|
||||
},
|
||||
"lafoirfouille.fr": {
|
||||
"search_url": "https://www.lafoirfouille.fr/recherche?q={query}",
|
||||
"product_selector": "a.product-item-link, .product-card a",
|
||||
"wait_selector": ".product-item, .products",
|
||||
"name": "La Foir'Fouille",
|
||||
"search_url": "https://www.lafoirfouille.fr/catalogsearch/result/?q={query}",
|
||||
"product_selector": (
|
||||
".product-item a.product-item-link, .product-item-info a, "
|
||||
".products-grid a, a[href$='.html']"
|
||||
),
|
||||
"wait_selector": ".products-grid, .product-items, .search-results",
|
||||
"category": "Discount",
|
||||
"requires_js": True,
|
||||
"priority": 7,
|
||||
},
|
||||
"tedi.fr": {
|
||||
"search_url": "https://www.tedi.fr/search?q={query}",
|
||||
"product_selector": "a.product-card, .product a",
|
||||
"wait_selector": ".product-card, .search-results",
|
||||
},
|
||||
"leclerc.fr": {
|
||||
# === GRANDES SURFACES ===
|
||||
"e.leclerc": {
|
||||
"name": "E.Leclerc",
|
||||
"search_url": "https://www.e.leclerc/recherche?q={query}",
|
||||
"product_selector": "a.product-card, .product-tile a",
|
||||
"wait_selector": ".product-card, .products-grid",
|
||||
},
|
||||
"carrefour.fr": {
|
||||
"search_url": "https://www.carrefour.fr/s?q={query}",
|
||||
"product_selector": "a.product-card, .product-item a",
|
||||
"wait_selector": ".product-card, .product-list",
|
||||
"product_selector": (
|
||||
"a[data-testid='product-card-link'], .product-card a, "
|
||||
".product-tile a, a[href*='/fp/']"
|
||||
),
|
||||
"wait_selector": ".product-card, .products-grid, [data-testid='product-card']",
|
||||
"category": "Grande Surface",
|
||||
"requires_js": True,
|
||||
"priority": 8,
|
||||
},
|
||||
"auchan.fr": {
|
||||
"name": "Auchan",
|
||||
"search_url": "https://www.auchan.fr/recherche?text={query}",
|
||||
"product_selector": "a.product-card, .product-tile a",
|
||||
"wait_selector": ".product-card, .products",
|
||||
"product_selector": (
|
||||
".product-card a, .product-tile a, "
|
||||
"a[href*='/p/'], .product-item a"
|
||||
),
|
||||
"wait_selector": ".product-card, .products, .product-list",
|
||||
"category": "Grande Surface",
|
||||
"requires_js": True,
|
||||
"priority": 9,
|
||||
},
|
||||
"carrefour.fr": {
|
||||
"name": "Carrefour",
|
||||
"search_url": "https://www.carrefour.fr/s?q={query}",
|
||||
"product_selector": (
|
||||
"a[data-testid='product-card-link'], .product-card a, "
|
||||
".product-item a, a[href*='/p/']"
|
||||
),
|
||||
"wait_selector": ".product-card, .product-list, [data-testid='product-card']",
|
||||
"category": "Grande Surface",
|
||||
"requires_js": True,
|
||||
"priority": 10,
|
||||
},
|
||||
# === E-COMMERCE GÉNÉRALISTE ===
|
||||
"amazon.fr": {
|
||||
"name": "Amazon France",
|
||||
"search_url": "https://www.amazon.fr/s?k={query}",
|
||||
"product_selector": "div[data-component-type='s-search-result'] h2 a",
|
||||
"wait_selector": "div[data-component-type='s-search-result']",
|
||||
"category": "E-commerce",
|
||||
"requires_js": True,
|
||||
"priority": 11,
|
||||
},
|
||||
"cdiscount.com": {
|
||||
"name": "Cdiscount",
|
||||
"search_url": "https://www.cdiscount.com/search/10/{query}.html",
|
||||
"product_selector": (
|
||||
"a.prdtBImg, .prdtBILDetails a, .product-item a, "
|
||||
"a[href*='/f-']"
|
||||
),
|
||||
"wait_selector": ".prdtBILDetails, .product-list",
|
||||
"category": "E-commerce",
|
||||
"requires_js": True,
|
||||
"priority": 12,
|
||||
},
|
||||
# === ÉLECTRONIQUE / HIGH-TECH ===
|
||||
"darty.com": {
|
||||
"name": "Darty",
|
||||
"search_url": "https://www.darty.com/nav/recherche?text={query}",
|
||||
"product_selector": (
|
||||
".product-card a, a[href*='/product'], "
|
||||
".product-list-item a"
|
||||
),
|
||||
"wait_selector": ".product-card, .product-list",
|
||||
"category": "Électronique",
|
||||
"requires_js": True,
|
||||
"priority": 13,
|
||||
},
|
||||
"boulanger.com": {
|
||||
"name": "Boulanger",
|
||||
"search_url": "https://www.boulanger.com/resultats?tr={query}",
|
||||
"product_selector": (
|
||||
".product-list__item a, a[href*='/ref/'], "
|
||||
".product-card a"
|
||||
),
|
||||
"wait_selector": ".product-list, .product-card",
|
||||
"category": "Électronique",
|
||||
"requires_js": True,
|
||||
"priority": 14,
|
||||
},
|
||||
# === AUTRES (conservés pour compatibilité) ===
|
||||
"amazon.com": {
|
||||
"name": "Amazon US",
|
||||
"search_url": "https://www.amazon.com/s?k={query}",
|
||||
"product_selector": "div[data-component-type='s-search-result'] h2 a",
|
||||
"wait_selector": "div[data-component-type='s-search-result']",
|
||||
"category": "E-commerce",
|
||||
"requires_js": True,
|
||||
"priority": 99,
|
||||
},
|
||||
"fnac.com": {
|
||||
"name": "Fnac",
|
||||
"search_url": "https://www.fnac.com/SearchResult/ResultList.aspx?Search={query}",
|
||||
"product_selector": "a.Article-title, .Article-item a.js-minifa-title",
|
||||
"wait_selector": ".Article-item",
|
||||
"category": "Culture & Tech",
|
||||
"requires_js": True,
|
||||
"priority": 15,
|
||||
},
|
||||
}
|
||||
|
||||
# Sélecteurs communs pour les bannières de cookies
|
||||
COOKIE_ACCEPT_SELECTORS = [
|
||||
# Boutons d'acceptation génériques
|
||||
"button[id*='accept']",
|
||||
"button[id*='cookie']",
|
||||
"button[class*='accept']",
|
||||
"button[class*='cookie']",
|
||||
"button[data-testid*='accept']",
|
||||
"button[data-testid*='cookie']",
|
||||
# Texte français
|
||||
"button:has-text('Accepter')",
|
||||
"button:has-text('J\\'accepte')",
|
||||
"button:has-text('Tout accepter')",
|
||||
"button:has-text('Accepter tout')",
|
||||
"button:has-text('Accepter les cookies')",
|
||||
"button:has-text('OK')",
|
||||
"button:has-text('Continuer')",
|
||||
# Texte anglais (certains sites utilisent l'anglais)
|
||||
"button:has-text('Accept')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('Accept cookies')",
|
||||
"button:has-text('I accept')",
|
||||
# Liens
|
||||
"a:has-text('Accepter')",
|
||||
"a:has-text('J\\'accepte')",
|
||||
"a:has-text('Tout accepter')",
|
||||
# Classes communes de CMP (Consent Management Platforms)
|
||||
".didomi-continue-without-agreeing",
|
||||
"#didomi-notice-agree-button",
|
||||
".onetrust-accept-btn-handler",
|
||||
"#onetrust-accept-btn-handler",
|
||||
".tarteaucitronAllow",
|
||||
"#tarteaucitronPersonalize",
|
||||
".cc-btn.cc-allow",
|
||||
".cky-btn-accept",
|
||||
"#CybsAccept",
|
||||
".qc-cmp2-summary-buttons button:first-child",
|
||||
"[data-accept-cookies]",
|
||||
"[data-gdpr-accept]",
|
||||
]
|
||||
|
||||
|
||||
async def _accept_cookies(page, domain: str) -> bool:
|
||||
"""
|
||||
Tente d'accepter les cookies sur une page.
|
||||
Retourne True si un bouton a été cliqué, False sinon.
|
||||
"""
|
||||
try:
|
||||
# Attendre un court moment pour que la bannière apparaisse
|
||||
await asyncio.sleep(1)
|
||||
|
||||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||||
try:
|
||||
button = page.locator(selector).first
|
||||
if await button.is_visible(timeout=500):
|
||||
await button.click(timeout=2000)
|
||||
logger.info(f"{domain}: Cookies acceptés avec sélecteur {selector}")
|
||||
# Attendre que la bannière disparaisse
|
||||
await asyncio.sleep(0.5)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
logger.debug(f"{domain}: Aucune bannière de cookies trouvée")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"{domain}: Erreur lors de l'acceptation des cookies: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def search(
|
||||
query: str,
|
||||
@@ -220,6 +386,9 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
|
||||
# Naviguer vers la page de recherche
|
||||
await page.goto(final_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
||||
|
||||
# Accepter les cookies si nécessaire
|
||||
await _accept_cookies(page, domain)
|
||||
|
||||
# Attendre que les résultats se chargent
|
||||
if wait_selector:
|
||||
try:
|
||||
|
||||
@@ -330,3 +330,96 @@ def delete_site(db: Session, site_id: int) -> bool:
|
||||
db.delete(site)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def seed_default_sites(db: Session) -> int:
|
||||
"""
|
||||
Initialise la base de données avec les sites configurés en dur.
|
||||
Ne crée que les sites qui n'existent pas encore (basé sur le domaine).
|
||||
|
||||
Returns:
|
||||
Nombre de sites créés
|
||||
"""
|
||||
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
|
||||
|
||||
created_count = 0
|
||||
|
||||
# Récupérer les domaines existants
|
||||
existing_domains = {site.domain.lower().replace("www.", "")
|
||||
for site in db.query(SearchSite).all()}
|
||||
|
||||
for domain, config in DEFAULT_SITE_CONFIGS.items():
|
||||
# Normaliser le domaine pour comparaison
|
||||
clean_domain = domain.lower().replace("www.", "")
|
||||
|
||||
# Ne pas créer si déjà existant
|
||||
if clean_domain in existing_domains:
|
||||
logger.debug(f"Site {domain} déjà existant, ignoré")
|
||||
continue
|
||||
|
||||
# Créer le site
|
||||
site_data = {
|
||||
"name": config.get("name", domain),
|
||||
"domain": domain,
|
||||
"search_url": config.get("search_url"),
|
||||
"product_link_selector": config.get("product_selector"),
|
||||
"category": config.get("category"),
|
||||
"requires_js": config.get("requires_js", True),
|
||||
"priority": config.get("priority", 99),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
try:
|
||||
site = SearchSite(**site_data)
|
||||
db.add(site)
|
||||
db.commit()
|
||||
created_count += 1
|
||||
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Erreur création site {domain}: {e}")
|
||||
|
||||
return created_count
|
||||
|
||||
|
||||
def reset_sites_to_defaults(db: Session) -> int:
|
||||
"""
|
||||
Réinitialise tous les sites avec les valeurs par défaut.
|
||||
Supprime tous les sites existants et recrée à partir de la config.
|
||||
|
||||
Returns:
|
||||
Nombre de sites créés
|
||||
"""
|
||||
from app.services.direct_search_service import DEFAULT_SITE_CONFIGS
|
||||
|
||||
# Supprimer tous les sites existants
|
||||
db.query(SearchSite).delete()
|
||||
db.commit()
|
||||
logger.info("Tous les sites supprimés")
|
||||
|
||||
# Recréer avec les valeurs par défaut
|
||||
created_count = 0
|
||||
|
||||
for domain, config in DEFAULT_SITE_CONFIGS.items():
|
||||
site_data = {
|
||||
"name": config.get("name", domain),
|
||||
"domain": domain,
|
||||
"search_url": config.get("search_url"),
|
||||
"product_link_selector": config.get("product_selector"),
|
||||
"category": config.get("category"),
|
||||
"requires_js": config.get("requires_js", True),
|
||||
"priority": config.get("priority", 99),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
try:
|
||||
site = SearchSite(**site_data)
|
||||
db.add(site)
|
||||
db.commit()
|
||||
created_count += 1
|
||||
logger.info(f"Site créé: {config.get('name', domain)} ({domain})")
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Erreur création site {domain}: {e}")
|
||||
|
||||
return created_count
|
||||
Reference in new issue
Block a user