mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: add tracking scraper service and define search configurations for various e-commerce sites.
This commit is contained in:
1 parent
cd72449a4e
commit
3b6c8527e3
2 files changed
+14
-3
No files matched your search
@@ -198,6 +198,7 @@ SITE_CONFIGS = {
|
||||
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
||||
"product_selector": "div[data-testid='product-card']",
|
||||
"product_image_selector": "img[data-testid='product-card-image']",
|
||||
"price_selector": ".product-price, [data-testid='product-price'], .price",
|
||||
"wait_selector": "div[data-testid='product-card']",
|
||||
"category": "Discount",
|
||||
"requires_proxy": True,
|
||||
|
||||
@@ -170,17 +170,27 @@ class ScraperService:
|
||||
return None, "", url, ""
|
||||
|
||||
try:
|
||||
# Determine Proxy Requirement
|
||||
# Determine Proxy & Selector Requirement
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(url).netloc.replace("www.", "")
|
||||
|
||||
# Check SITE_CONFIGS for proxy requirement
|
||||
# Check SITE_CONFIGS for proxy requirement and default selector
|
||||
requires_proxy = False
|
||||
config_selector = None
|
||||
|
||||
if domain in SITE_CONFIGS:
|
||||
requires_proxy = SITE_CONFIGS[domain].get("requires_proxy", False)
|
||||
config_selector = SITE_CONFIGS[domain].get("price_selector")
|
||||
elif f"www.{domain}" in SITE_CONFIGS: # Fallback check
|
||||
requires_proxy = SITE_CONFIGS[f"www.{domain}"].get("requires_proxy", False)
|
||||
cfg = SITE_CONFIGS[f"www.{domain}"]
|
||||
requires_proxy = cfg.get("requires_proxy", False)
|
||||
config_selector = cfg.get("price_selector")
|
||||
|
||||
# Use config selector if none provided
|
||||
if selector is None and config_selector:
|
||||
logger.info(f"Using configured price selector for {domain}: {config_selector}")
|
||||
selector = config_selector
|
||||
|
||||
# Special override: If Action.com, force proxy
|
||||
if "action.com" in domain:
|
||||
requires_proxy = True
|
||||
|
||||
Reference in new issue
Block a user