Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-13 18:26:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from R0m1k3/claude/hardcode-website-urls-01QrWZK7suQnuBrxYgK1r5JC

Browse Source 
Claude/hardcode website urls 01 qr wzk7su qnu brx yg k1r5 jc
This commit is contained in:
LogiFlow authored and GitHub committed 2025-11-23 09:16:57 +01:00
2 parents 2484ec66a3 00d3aa9f1e
commit 356c5a6ef4
3 files changed
+148 -34
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/ai_schema.py
+32 -14
View File
@@ -73,14 +73,28 @@ class AIExtractionResponse(BaseModel):
@field_validator("price", mode="before")
@classmethod
def normalize_price(cls, v):
"""Normalize price to float or None."""
"""Normalize price to float or None, handling French format."""
if v is None or v in ("null", ""):
return None
if isinstance(v, str):
# Remove currency symbols and commas
cleaned = re.sub(r"[^\d.]", "", v)
# Handle French format: "1 234,56" -> "1234.56"
# First, remove spaces (thousand separators)
cleaned = v.replace(" ", "").replace("\u00a0", "")
# Replace comma with dot (French decimal separator)
# But only if there's no dot already (to handle "1.234,56" format)
if "," in cleaned and "." in cleaned:
# Format "1.234,56" - remove dots, replace comma with dot
cleaned = cleaned.replace(".", "").replace(",", ".")
elif "," in cleaned:
# Format "12,99" - just replace comma with dot
cleaned = cleaned.replace(",", ".")
# Remove currency symbols and other non-numeric chars (except dot)
cleaned = re.sub(r"[^\d.]", "", cleaned)
if cleaned:
return float(cleaned)
try:
return float(cleaned)
except ValueError:
return None
return None
return float(v)
@@ -116,23 +130,27 @@ class AIExtractionMetadata(BaseModel):
# Prompt template for schema-first extraction
EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from the image.
EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from this French e-commerce page.
**PRICE:**
- Find the main current price
- Ignore crossed-out prices
- Extract number only (no symbols)
**PRICE (IMPORTANT - French format):**
- French prices use COMMA for decimals: "12,99 €" means 12.99
- Thousands separator is SPACE or DOT: "1 234,56 €" or "1.234,56 €" means 1234.56
- Currency symbol is usually € at the end
- Look for: price tags, "Prix:", "€", numbers near "Ajouter au panier"
- Extract as DECIMAL NUMBER (convert comma to dot): 12,99 -> 12.99
- Ignore crossed-out/barré prices (old prices)
- If multiple prices, take the current/main price (not the original)
- If unclear: set null and confidence < 0.5
**STOCK:**
- TRUE if: "Add to Cart", "Buy Now", "In Stock", "Available"
- FALSE if: "Out of Stock", "Sold Out", "Unavailable", "Notify Me"
- TRUE if: "Ajouter au panier", "Acheter", "En stock", "Disponible", "Add to Cart"
- FALSE if: "Rupture", "Indisponible", "Épuisé", "Out of Stock", "Notify Me"
- NULL if unclear or not shown
**CONFIDENCE (0.0 to 1.0):**
- 0.9-1.0: Very certain
- 0.5-0.8: Moderately certain
- Below 0.5: Unsure
- 0.9-1.0: Price clearly visible
- 0.5-0.8: Price found but partially obscured or uncertain
- Below 0.5: Cannot find price reliably
Respond ONLY with valid JSON:
{{
app/services/direct_search_service.py
+64 -14
View File
@@ -166,15 +166,22 @@ DEFAULT_SITE_CONFIGS = {
# === E-COMMERCE GÉNÉRALISTE ===
"amazon.fr": {
"name": "Amazon France",
"search_url": "https://www.amazon.fr/s?k={query}&language=fr_FR",
# URL de recherche Amazon avec paramètres pour éviter les redirections
"search_url": "https://www.amazon.fr/s?k={query}&ref=nb_sb_noss",
"product_selector": (
"div[data-component-type='s-search-result'] h2 a, "
".s-result-item h2 a, [data-asin] h2 a"
# Sélecteurs multiples pour robustesse
"div[data-component-type='s-search-result'] h2 a.a-link-normal, "
"div[data-asin] h2 a.a-link-normal, "
".s-result-item h2 a.a-link-normal, "
"[data-asin]:not([data-asin='']) h2 a, "
".s-main-slot .s-result-item a.a-link-normal.s-no-outline"
),
"wait_selector": "div[data-component-type='s-search-result'], .s-result-item",
"wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item",
"category": "E-commerce",
"requires_js": True,
"priority": 11,
# Amazon nécessite une attention particulière pour les cookies
"needs_cookie_accept": True,
},
"cdiscount.com": {
"name": "Cdiscount",
@@ -216,12 +223,18 @@ DEFAULT_SITE_CONFIGS = {
# === AUTRES (conservés pour compatibilité) ===
"amazon.com": {
"name": "Amazon US",
"search_url": "https://www.amazon.com/s?k={query}",
"product_selector": "div[data-component-type='s-search-result'] h2 a",
"wait_selector": "div[data-component-type='s-search-result']",
"search_url": "https://www.amazon.com/s?k={query}&ref=nb_sb_noss",
"product_selector": (
"div[data-component-type='s-search-result'] h2 a.a-link-normal, "
"div[data-asin] h2 a.a-link-normal, "
".s-result-item h2 a.a-link-normal, "
"[data-asin]:not([data-asin='']) h2 a"
),
"wait_selector": "[data-component-type='s-search-result'], .s-main-slot .s-result-item",
"category": "E-commerce",
"requires_js": True,
"priority": 99,
"needs_cookie_accept": True,
},
"fnac.com": {
"name": "Fnac",
@@ -436,21 +449,51 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
logger.info(f"Recherche Browserless sur {domain}: {final_url}")
# Détecter si c'est Amazon (protection anti-bot forte)
is_amazon = "amazon" in domain
try:
async with async_playwright() as p:
# Se connecter à Browserless
browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
try:
context = await browser.new_context(
viewport={"width": 1920, "height": 1080},
user_agent=USER_AGENT,
locale="fr-FR",
)
# Configuration spéciale pour Amazon
if is_amazon:
context = await browser.new_context(
viewport={"width": 1920, "height": 1080},
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
),
locale="fr-FR",
timezone_id="Europe/Paris",
extra_http_headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
},
)
else:
context = await browser.new_context(
viewport={"width": 1920, "height": 1080},
user_agent=USER_AGENT,
locale="fr-FR",
)
page = await context.new_page()
# Bloquer les ressources inutiles pour accélérer
await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
# Pour Amazon, ne pas bloquer les images (détection anti-bot)
if not is_amazon:
await page.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
await page.route("**/analytics*", lambda route: route.abort())
await page.route("**/tracking*", lambda route: route.abort())
@@ -460,6 +503,13 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
# Accepter les cookies si nécessaire
await _accept_cookies(page, domain)
# Pour Amazon, attendre plus longtemps et scroller
if is_amazon:
await asyncio.sleep(3)
# Scroll pour charger le contenu lazy-loaded
await page.evaluate("window.scrollBy(0, 500)")
await asyncio.sleep(1)
# Attendre que les résultats se chargent
if wait_selector:
try:
app/services/scraper_service.py
+52 -6
View File
@@ -209,16 +209,64 @@ class ScraperService:
# Wait a bit for dynamic content if needed
await page.wait_for_timeout(2000)
# Try to close common popups (inclut Amazon)
logger.info("Attempting to close popups...")
# Try to close common popups and cookie banners
logger.info("Attempting to close popups and cookie banners...")
popup_selectors = [
# Amazon spécifique - boutons cookies
# === SITES SPÉCIFIQUES FRANÇAIS ===
# Amazon France
"#sp-cc-accept",
"#sp-cc-rejectall-link",
"input[data-action-type='DISMISS']",
"#a-popover-content-1 button",
"[data-action='a-popover-close']",
# Popups généraux
# E.Leclerc, Darty, Fnac
"#onetrust-accept-btn-handler",
".onetrust-accept-btn-handler",
# Auchan
"#popin_tc_privacy_button_2",
".popin_tc_privacy_button",
"#didomi-notice-agree-button",
# Carrefour
"[data-testid='accept-cookies-button']",
# Cdiscount
"#footer_tc_privacy_button_2",
".privacy_prompt_accept",
# Boulanger
".bcom-consent-accept-all",
"#cookieBanner-accept",
# Gifi, Centrakor, La Foir'Fouille (Magento)
".action-primary.action-accept",
"#btn-cookie-allow",
# PrestaShop (Stokomani, B&M, etc.)
".btn-primary[data-dismiss='modal']",
"#gdpr_consent_agree",
# === CMP (Consent Management Platforms) ===
".didomi-continue-without-agreeing",
"#tarteaucitronPersonalize2",
".tarteaucitronAllow",
"#tarteaucitronAllDenied2",
".cc-btn.cc-allow",
".cky-btn-accept",
"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
"#CybotCookiebotDialogBodyButtonAccept",
".qc-cmp2-summary-buttons button:first-child",
"#axeptio_btn_acceptAll",
".axeptio_acceptAll",
# === TEXTE FRANÇAIS ===
"button:has-text('Tout accepter')",
"button:has-text('Accepter tout')",
"button:has-text('Accepter et fermer')",
"button:has-text('Accepter les cookies')",
"button:has-text('Accepter')",
"button:has-text('J\\'accepte')",
"button:has-text('OK')",
"button:has-text('Continuer')",
"a:has-text('Tout accepter')",
"a:has-text('Accepter')",
# === POPUPS GÉNÉRAUX ===
"button[aria-label='Close']",
"button[aria-label='close']",
"button[aria-label='Fermer']",
@@ -229,9 +277,7 @@ class ScraperService:
"[class*='modal'] button",
"button:has-text('No, thanks')",
"button:has-text('No thanks')",
"button:has-text('Accepter')",
"button:has-text('Accept')",
"button:has-text('Continuer')",
"a:has-text('No, thanks')",
"div[role='dialog'] button[aria-label='Close']",
]