mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-23 23:28:12 +02:00
feat: Add initial search configuration for e-commerce sites and a tracking scraper service.
This commit is contained in:
1 parent
3b6c8527e3
commit
844c75f190
2 files changed
+27
-5
No files matched your search
@@ -198,10 +198,10 @@ SITE_CONFIGS = {
|
|||||||
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
"search_url": "https://www.action.com/fr-fr/search/?q={query}",
|
||||||
"product_selector": "div[data-testid='product-card']",
|
"product_selector": "div[data-testid='product-card']",
|
||||||
"product_image_selector": "img[data-testid='product-card-image']",
|
"product_image_selector": "img[data-testid='product-card-image']",
|
||||||
"price_selector": ".product-price, [data-testid='product-price'], .price",
|
"price_selector": ".product-price, [data-testid='product-price'], .price, .current-price, .product-card__price",
|
||||||
"wait_selector": "div[data-testid='product-card']",
|
"wait_selector": "div[data-testid='product-card']",
|
||||||
"category": "Discount",
|
"category": "Discount",
|
||||||
"requires_proxy": True,
|
"requires_proxy": False,
|
||||||
},
|
},
|
||||||
"e-leclerc.com": {
|
"e-leclerc.com": {
|
||||||
"name": "E.Leclerc",
|
"name": "E.Leclerc",
|
||||||
|
|||||||
@@ -191,9 +191,9 @@ class ScraperService:
|
|||||||
logger.info(f"Using configured price selector for {domain}: {config_selector}")
|
logger.info(f"Using configured price selector for {domain}: {config_selector}")
|
||||||
selector = config_selector
|
selector = config_selector
|
||||||
|
|
||||||
# Special override: If Action.com, force proxy
|
# Special override: If Action.com, force proxy ONLY if strictly needed and proxies are available
|
||||||
if "action.com" in domain:
|
# if "action.com" in domain:
|
||||||
requires_proxy = True
|
# requires_proxy = True
|
||||||
|
|
||||||
proxy = None
|
proxy = None
|
||||||
if requires_proxy:
|
if requires_proxy:
|
||||||
@@ -208,6 +208,10 @@ class ScraperService:
|
|||||||
if "amazon" in url:
|
if "amazon" in url:
|
||||||
return await ScraperService._scrape_amazon_specific(url, item_id, config, return_html, proxy)
|
return await ScraperService._scrape_amazon_specific(url, item_id, config, return_html, proxy)
|
||||||
|
|
||||||
|
# SPECIALIZED ACTION HANDLING (Rupture de stock check)
|
||||||
|
is_action = "action.com" in url
|
||||||
|
|
||||||
|
|
||||||
context = await ScraperService._create_context(ScraperService._browser, url, proxy=proxy)
|
context = await ScraperService._create_context(ScraperService._browser, url, proxy=proxy)
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
|
|
||||||
@@ -698,6 +702,24 @@ class ScraperService:
|
|||||||
}
|
}
|
||||||
""")
|
""")
|
||||||
|
|
||||||
|
# Extra cleanup to prevent TitlePrice merging
|
||||||
|
# If we are on Action, we might need specific spacing logic
|
||||||
|
if "action.com" in page.url:
|
||||||
|
clean_text = await page.evaluate("""
|
||||||
|
() => {
|
||||||
|
const title = document.querySelector('h1, .product-title');
|
||||||
|
const price = document.querySelector('.price, .product-price');
|
||||||
|
|
||||||
|
let text = document.body.innerText;
|
||||||
|
|
||||||
|
// Manually construct clean string if elements found
|
||||||
|
if(title && price) {
|
||||||
|
return title.innerText + " " + price.innerText + "\\n" + text;
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
|
||||||
# Fallback if cleaning removed everything (unlikely but safe)
|
# Fallback if cleaning removed everything (unlikely but safe)
|
||||||
if not clean_text or len(clean_text) < 100:
|
if not clean_text or len(clean_text) < 100:
|
||||||
logger.warning("Cleaned text too short, falling back to full body text")
|
logger.warning("Cleaned text too short, falling back to full body text")
|
||||||
|
|||||||
Reference in new issue
Block a user