feat: add tracking scraper service and define search configurations for various e-commerce sites.

This commit is contained in:
Michael committed 2026-01-30 14:39:53 +01:00
1 parent cd72449a4e
commit 3b6c8527e3
2 files changed
+14 -3

No files matched your search

+1
View File
@@ -198,6 +198,7 @@ SITE_CONFIGS = {
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
"product_selector": "div[data-testid='product-card']",
"product_image_selector": "img[data-testid='product-card-image']",
"price_selector": ".product-price, [data-testid='product-price'], .price",
"wait_selector": "div[data-testid='product-card']",
"category": "Discount",
"requires_proxy": True,
+13 -3
View File
@@ -170,17 +170,27 @@ class ScraperService:
return None, "", url, ""
try:
# Determine Proxy Requirement
# Determine Proxy & Selector Requirement
from urllib.parse import urlparse
domain = urlparse(url).netloc.replace("www.", "")
# Check SITE_CONFIGS for proxy requirement
# Check SITE_CONFIGS for proxy requirement and default selector
requires_proxy = False
config_selector = None
if domain in SITE_CONFIGS:
requires_proxy = SITE_CONFIGS[domain].get("requires_proxy", False)
config_selector = SITE_CONFIGS[domain].get("price_selector")
elif f"www.{domain}" in SITE_CONFIGS: # Fallback check
requires_proxy = SITE_CONFIGS[f"www.{domain}"].get("requires_proxy", False)
cfg = SITE_CONFIGS[f"www.{domain}"]
requires_proxy = cfg.get("requires_proxy", False)
config_selector = cfg.get("price_selector")
# Use config selector if none provided
if selector is None and config_selector:
logger.info(f"Using configured price selector for {domain}: {config_selector}")
selector = config_selector
# Special override: If Action.com, force proxy
if "action.com" in domain:
requires_proxy = True