From 1893988bb1d8d44d6f54026c90f983880480bfd5 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Thu, 8 Jan 2026 10:42:15 +0100 Subject: [PATCH] modf --- app/schemas.py | 6 + app/services/scheduler_service.py | 135 +++++++++++++----- .../src/components/dashboard/ItemCard.jsx | 22 ++- 3 files changed, 128 insertions(+), 35 deletions(-) diff --git a/app/schemas.py b/app/schemas.py index 9e510e1..ab13df8 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -19,6 +19,7 @@ class ItemResponse(ItemCreate): id: int current_price: float | None in_stock: bool | None + is_available: bool | None = True # Product still exists on retailer site current_price_confidence: float | None = None in_stock_confidence: float | None = None is_active: bool @@ -45,6 +46,7 @@ class SettingsUpdate(BaseModel): # === Search Sites Schemas === + class SearchSiteCreate(BaseModel): name: str domain: str @@ -93,6 +95,7 @@ class SearchSiteResponse(BaseModel): # === Search Results Schemas === + class SearchQuery(BaseModel): query: str site_ids: list[int] | None = None # If None, use all active sites @@ -101,6 +104,7 @@ class SearchQuery(BaseModel): class SearchResultItem(BaseModel): """Un produit trouvé lors de la recherche""" + url: str title: str price: float | None = None @@ -115,6 +119,7 @@ class SearchResultItem(BaseModel): class SearchProgress(BaseModel): """Progression de la recherche pour SSE""" + status: str # "searching", "scraping", "completed", "error" total: int completed: int @@ -126,6 +131,7 @@ class SearchProgress(BaseModel): # === Notification Schemas === + class NotificationChannelCreate(BaseModel): name: str type: str # "email", "discord", "mattermost" diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index a0a8229..98b0798 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -93,6 +93,35 @@ def _update_db_error(item_id, error_msg): session.commit() +def _update_db_product_unavailable(item_id: int, error_msg: str, screenshot_path: str | None = None): + """Mark item as unavailable (product no longer exists on retailer site).""" + with database.SessionLocal() as session: + if item := session.query(models.Item).filter(models.Item.id == item_id).first(): + item.is_refreshing = False + item.is_available = False # Mark product as unavailable + item.in_stock = False # Also mark as out of stock + item.last_error = error_msg + item.last_checked = datetime.now(UTC) + + # Record in history if we have a screenshot + if screenshot_path: + session.add( + models.PriceHistory( + item_id=item.id, + price=item.current_price or 0, # Keep last known price + screenshot_path=screenshot_path, + price_confidence=0.0, # Zero confidence = unavailable + in_stock_confidence=1.0, # High confidence it's unavailable + ai_model="unavailable-detector", + ai_provider="internal", + prompt_version="unavailable-v1", + repair_used=False, + ) + ) + + session.commit() + + async def process_item_check(item_id: int): loop = asyncio.get_running_loop() with database.SessionLocal() as session: @@ -106,12 +135,9 @@ async def process_item_check(item_id: int): logger.info(f"Checking item: {item_data['name']} ({item_data['url']})") # Use independent ScraperService for tracking from app.services.tracking_scraper_service import ScraperService - + screenshot_path, html_content, final_url, page_title = await ScraperService.scrape_item( - url=item_data["url"], - selector=item_data["selector"], - item_id=item_id, - return_html=True + url=item_data["url"], selector=item_data["selector"], item_id=item_id, return_html=True ) if not screenshot_path: @@ -120,30 +146,67 @@ async def process_item_check(item_id: int): # Detect Amazon login wall/redirection/block is_amazon = "amazon" in item_data["url"] is_blocked = False - + if is_amazon: login_terms = ["signin", "captcha", "s'identifier", "log in", "login"] title_lower = page_title.lower() - if any(term in final_url.lower() for term in login_terms) or \ - any(term in title_lower for term in login_terms) or \ - "amazon.fr: s'identifier" in title_lower: + if ( + any(term in final_url.lower() for term in login_terms) + or any(term in title_lower for term in login_terms) + or "amazon.fr: s'identifier" in title_lower + ): is_blocked = True - + if is_blocked: logger.warning(f"Amazon Bot Detection triggered for item {item_id}. Title: {page_title}, URL: {final_url}") await loop.run_in_executor(None, _update_db_error, item_id, f"Amazon Bot Detection: {page_title}") return + # Detect Action.com product unavailability + is_action = "action.com" in item_data["url"] + is_product_unavailable = False + + if is_action: + unavailable_terms = [ + "showproductwarning=true", + "produit indisponible", + "malheureusement, ce produit est actuellement indisponible", + ] + final_url_lower = final_url.lower() + html_lower = html_content.lower() if html_content else "" + + if any(term in final_url_lower for term in unavailable_terms) or any( + term in html_lower for term in unavailable_terms + ): + is_product_unavailable = True + + if is_product_unavailable: + logger.warning(f"Product unavailable detected for item {item_id} on Action.com") + await loop.run_in_executor( + None, _update_db_product_unavailable, item_id, "Produit indisponible sur Action.com", screenshot_path + ) + + # Send notification if configured + if item_data["notification_channel"]: + await NotificationService.send_notification( + item_data["notification_channel"], + title=f"⚠️ Produit indisponible : {item_data['name']}", + body=f"Le produit '{item_data['name']}' n'est plus disponible sur Action.com.\n{item_data['url']}", + ) + return + # HYBRID EXTRACTION STRATEGY (Aligned with ImprovedSearchService) # 1. Try specialized parser (if available) or JSON-LD price = None in_stock = True - + # Site-specific parser (Gifi) from urllib.parse import urlparse + domain = urlparse(item_data["url"]).netloc if "gifi.fr" in domain: from app.services.parsers.gifi_parser import GifiParser + try: parser = GifiParser() details = parser.parse_product_details(html_content, item_data["url"]) @@ -157,22 +220,27 @@ async def process_item_check(item_id: int): # Simple JSON-LD extract (if not found by specific parser) if price is None: import json + try: from bs4 import BeautifulSoup + soup = BeautifulSoup(html_content, "html.parser") scripts = soup.find_all("script", type="application/ld+json") for script in scripts: if script.string: try: data = json.loads(script.string) - if isinstance(data, list): data = data[0] + if isinstance(data, list): + data = data[0] if data.get("@type") == "Product" and "offers" in data: offers = data["offers"] - if isinstance(offers, list) and offers: offers = offers[0] + if isinstance(offers, list) and offers: + offers = offers[0] if "price" in offers: - price = float(str(offers["price"]).replace(',', '.')) + price = float(str(offers["price"]).replace(",", ".")) break - except: pass + except: + pass except Exception as e: logger.debug(f"JSON-LD extraction failed: {e}") @@ -187,26 +255,23 @@ async def process_item_check(item_id: int): # 3. Fallback/Verification with AIService (Vision AI - Gemini) extraction = None metadata = None - + if price is not None: # Create a synthetic AI response if we already have a high-confidence price extraction = AIExtractionResponse( - price=price, - in_stock=in_stock, - price_confidence=0.95, - in_stock_confidence=0.9, - source_type="text" + price=price, in_stock=in_stock, price_confidence=0.95, in_stock_confidence=0.9, source_type="text" ) metadata = AIExtractionMetadata( model_name="hybrid-text-parser", provider="internal-hybrid", prompt_version="hybrid-v2", - repair_used=False + repair_used=False, ) else: # Fallback to Vision AI if text extraction failed # Clean text properly before sending to AI from app.utils.text import clean_text + cleaned_html = clean_text(html_content) if not (ai_result := await AIService.analyze_image(screenshot_path, page_text=cleaned_html[:10000])): raise Exception("AI analysis (Vision) failed") @@ -220,25 +285,29 @@ async def process_item_check(item_id: int): # Check for notifications if item_data["notification_channel"] and extraction.price: channel = item_data["notification_channel"] - logger.info(f"📢 Notification check for item {item_id}: old_price={old_price}, new_price={extraction.price}, target_price={item_data['target_price']}") - + logger.info( + f"📢 Notification check for item {item_id}: old_price={old_price}, new_price={extraction.price}, target_price={item_data['target_price']}" + ) + notification_sent = False - + # CASE 1: Target price is set -> notify only when target is reached if item_data["target_price"]: if extraction.price <= item_data["target_price"]: # Only notify if we haven't already notified for this price if not old_price or old_price > item_data["target_price"]: - logger.info(f"✅ Target price reached for {item_data['name']}: {extraction.price}€ <= {item_data['target_price']}€") + logger.info( + f"✅ Target price reached for {item_data['name']}: {extraction.price}€ <= {item_data['target_price']}€" + ) await NotificationService.send_notification( channel, title=f"🎯 Prix cible atteint : {item_data['name']}", - body=f"Le prix de {item_data['name']} est passé à {extraction.price}€ (Cible: {item_data['target_price']}€)\n{item_data['url']}" + body=f"Le prix de {item_data['name']} est passé à {extraction.price}€ (Cible: {item_data['target_price']}€)\n{item_data['url']}", ) notification_sent = True else: logger.info(f"ℹ️ Price {extraction.price}€ not yet at target {item_data['target_price']}€") - + # CASE 2: No target price -> notify on ANY price drop else: if old_price and extraction.price < old_price: @@ -247,10 +316,10 @@ async def process_item_check(item_id: int): await NotificationService.send_notification( channel, title=f"📉 Baisse de prix : {item_data['name']}", - body=f"Le prix de {item_data['name']} a baissé de {drop_percent:.1f}% !\nNouveau prix : {extraction.price}€ (Ancien: {old_price}€)\n{item_data['url']}" + body=f"Le prix de {item_data['name']} a baissé de {drop_percent:.1f}% !\nNouveau prix : {extraction.price}€ (Ancien: {old_price}€)\n{item_data['url']}", ) notification_sent = True - + # Also notify on significant price increases (>= 5%) elif old_price and extraction.price > old_price: increase_percent = (extraction.price - old_price) / old_price * 100 @@ -259,10 +328,10 @@ async def process_item_check(item_id: int): await NotificationService.send_notification( channel, title=f"📈 Hausse de prix : {item_data['name']}", - body=f"Le prix de {item_data['name']} a augmenté de {increase_percent:.1f}%.\nNouveau prix : {extraction.price}€ (Ancien: {old_price}€)\n{item_data['url']}" + body=f"Le prix de {item_data['name']} a augmenté de {increase_percent:.1f}%.\nNouveau prix : {extraction.price}€ (Ancien: {old_price}€)\n{item_data['url']}", ) notification_sent = True - + if not notification_sent: logger.info(f"ℹ️ No notification triggered for item {item_id} (no significant change)") else: @@ -271,8 +340,6 @@ async def process_item_check(item_id: int): if not extraction.price: logger.debug(f"⚠️ Item {item_id} has no extracted price.") - - except Exception as e: logger.error(f"Error in process_item_check: {e}") await loop.run_in_executor(None, _update_db_error, item_id, str(e)) diff --git a/frontend/src/components/dashboard/ItemCard.jsx b/frontend/src/components/dashboard/ItemCard.jsx index 73e9e75..3484587 100644 --- a/frontend/src/components/dashboard/ItemCard.jsx +++ b/frontend/src/components/dashboard/ItemCard.jsx @@ -24,8 +24,16 @@ export function ItemCard({ item, onEdit, onDelete, onCheck, onZoom, onCategoryUp const stockStatus = getStockStatus(item.in_stock); + // Detect if product is no longer available on retailer site + const isProductUnavailable = item.is_available === false; + return ( - +
onZoom(item.screenshot_url)}> {item.screenshot_url ? ( <> @@ -61,6 +69,18 @@ export function ItemCard({ item, onEdit, onDelete, onCheck, onZoom, onCategoryUp
{stockStatus.label}
+ + {/* Product Unavailable Overlay */} + {isProductUnavailable && ( +
+
+ + + Produit retiré du site + +
+
+ )}