Add files via upload

app/services/ai_service.py

@@ -0,0 +1,352 @@
+import asyncio
+import json
+import logging
+import re
+import time
+from typing import Any, TypedDict
+
+from litellm import acompletion
+from pydantic import ValidationError
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from app import models
+from app.ai_schema import (
+    PROMPT_VERSION,
+    AIExtractionMetadata,
+    AIExtractionResponse,
+    get_extraction_prompt,
+    get_repair_prompt,
+)
+from app.database import SessionLocal
+from app.utils.image import encode_image
+from app.utils.text import clean_text
+
+# Default configuration (can be overridden by DB settings)
+DEFAULT_PROVIDER = "ollama"
+DEFAULT_MODEL = "gemma3:4b"
+DEFAULT_API_BASE = "http://ollama:11434"
+DEFAULT_TEMPERATURE = 0.1
+DEFAULT_MAX_TOKENS = 300
+DEFAULT_TIMEOUT = 30  # seconds
+MAX_TEXT_LENGTH = 5000  # Will be filtered to ~1500-2000 relevant chars
+
+# Config caching
+_config_cache: dict[str, Any] = {"data": None, "timestamp": 0.0}
+CONFIG_CACHE_TTL = 60  # seconds
+
+logger = logging.getLogger(__name__)
+
+
+class AIConfig(TypedDict):
+    provider: str
+    model: str
+    api_key: str
+    api_base: str
+    temperature: float
+    max_tokens: int
+    timeout: int
+    enable_json_repair: bool
+    enable_multi_sample: bool
+    multi_sample_threshold: float
+
+
+MIN_API_KEY_LENGTH = 12
+
+
+def _sanitize_api_key(key: str) -> str:
+    """Redact API key for logging (show first/last 4 chars only)."""
+    if not key or len(key) < MIN_API_KEY_LENGTH:
+        return "***"
+    return f"{key[:4]}...{key[-4:]}"
+
+
+class AIService:
+    @staticmethod
+    def get_ai_config() -> AIConfig:
+        """
+        Fetches AI configuration from the database with caching.
+        Returns a dict with: provider, model, api_key, api_base, temperature, max_tokens, timeout
+        """
+        # Check cache
+        now = time.time()
+        if _config_cache["data"] and (now - _config_cache["timestamp"]) < CONFIG_CACHE_TTL:
+            return _config_cache["data"]  # type: ignore
+
+        # Fetch from DB
+        session = SessionLocal()
+        try:
+            settings = session.query(models.Settings).all()
+            settings_map = {s.key: s.value for s in settings}
+
+            config: AIConfig = {
+                "provider": settings_map.get("ai_provider", DEFAULT_PROVIDER),
+                "model": settings_map.get("ai_model", DEFAULT_MODEL),
+                "api_key": settings_map.get("ai_api_key", ""),
+                "api_base": settings_map.get("ai_api_base", DEFAULT_API_BASE),
+                "temperature": float(settings_map.get("ai_temperature", str(DEFAULT_TEMPERATURE))),
+                "max_tokens": int(settings_map.get("ai_max_tokens", str(DEFAULT_MAX_TOKENS))),
+                "timeout": int(settings_map.get("ai_timeout", str(DEFAULT_TIMEOUT))),
+                "enable_json_repair": settings_map.get("enable_json_repair", "true").lower() == "true",
+                "enable_multi_sample": settings_map.get("enable_multi_sample", "false").lower() == "true",
+                "multi_sample_threshold": float(settings_map.get("multi_sample_confidence_threshold", "0.6")),
+            }
+
+            # Update cache
+            _config_cache["data"] = config  # type: ignore
+            _config_cache["timestamp"] = now
+
+            return config
+        except Exception as e:
+            logger.error(f"Error fetching AI config: {e}")
+            return {
+                "provider": DEFAULT_PROVIDER,
+                "model": DEFAULT_MODEL,
+                "api_key": "",
+                "api_base": DEFAULT_API_BASE,
+                "temperature": DEFAULT_TEMPERATURE,
+                "max_tokens": DEFAULT_MAX_TOKENS,
+                "timeout": DEFAULT_TIMEOUT,
+                "enable_json_repair": True,
+                "enable_multi_sample": False,
+                "multi_sample_threshold": 0.6,
+            }
+        finally:
+            session.close()
+
+    @staticmethod
+    def parse_and_validate_response(response_text: str) -> AIExtractionResponse:
+        """
+        Parse and validate AI response against schema.
+
+        Pipeline:
+        1. Extract JSON from response (handle markdown code blocks)
+        2. Parse JSON
+        3. Validate against Pydantic schema (includes normalization and clamping)
+
+        Raises:
+            ValidationError: If response doesn't match schema
+            json.JSONDecodeError: If JSON is invalid
+        """
+        # Try to extract JSON from markdown code blocks
+        json_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", response_text, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(1)
+        else:
+            # Try to find raw JSON object
+            json_match = re.search(r"\{.*\}", response_text, re.DOTALL)
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = response_text
+
+        # Parse JSON
+        data = json.loads(json_str)
+
+        # Validate and normalize through Pydantic
+        return AIExtractionResponse(**data)
+
+    @classmethod
+    async def repair_json_response(
+        cls,
+        raw_output: str,
+        config: AIConfig,
+    ) -> AIExtractionResponse:
+        """
+        Attempt to repair invalid JSON using a second LLM call.
+
+        Args:
+            raw_output: The raw AI output that failed parsing
+            config: AI configuration dict
+
+        Returns:
+            Validated AIExtractionResponse
+
+        Raises:
+            Exception: If repair also fails
+        """
+        logger.warning("Attempting JSON repair with second LLM call")
+
+        repair_prompt = get_repair_prompt(raw_output)
+
+        # Use a simpler, cheaper model for repair if possible
+        # For now, use the same model
+        kwargs = {
+            "model": config["model"] if config["provider"] != "ollama" else f"ollama/{config['model']}",
+            "messages": [{"role": "user", "content": repair_prompt}],
+            "max_tokens": 300,
+            "temperature": 0.0,  # Very deterministic for repair
+            "timeout": config["timeout"],
+        }
+
+        if config["api_key"]:
+            kwargs["api_key"] = config["api_key"]
+
+        if config["provider"] == "ollama":
+            kwargs["api_base"] = config["api_base"]
+            kwargs["format"] = "json"
+        elif config["provider"] == "openai" and config["api_base"]:
+            kwargs["api_base"] = config["api_base"]
+
+        response = await acompletion(**kwargs)
+        repaired_text = response.choices[0].message.content
+
+        # Validate repaired response
+        return cls.parse_and_validate_response(repaired_text)
+
+    @staticmethod
+    @retry(
+        retry=retry_if_exception_type((TimeoutError, ConnectionError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def call_llm(
+        prompt: str,
+        image_data_url: str,
+        config: AIConfig,
+    ) -> str:
+        """
+        Call LLM with structured output settings and retry logic.
+
+        Returns:
+            Raw response text from the model
+
+        Raises:
+            TimeoutError: If request times out after retries
+            ConnectionError: If connection fails after retries
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {"type": "image_url", "image_url": {"url": image_data_url}},
+                ],
+            }
+        ]
+
+        # Prepare kwargs for litellm
+        kwargs: dict[str, Any] = {
+            "model": config["model"] if config["provider"] != "ollama" else f"ollama/{config['model']}",
+            "messages": messages,
+            "max_tokens": config["max_tokens"],
+            "temperature": config["temperature"],
+            "timeout": config["timeout"],
+        }
+
+        if config["api_key"]:
+            kwargs["api_key"] = config["api_key"]
+
+        # Provider-specific structured output features
+        if config["provider"] == "ollama":
+            kwargs["api_base"] = config["api_base"]
+            kwargs["format"] = "json"  # Force JSON mode for Ollama
+        elif config["provider"] == "openai":
+            # Use OpenAI's JSON mode
+            kwargs["response_format"] = {"type": "json_object"}
+            if config["api_base"]:
+                kwargs["api_base"] = config["api_base"]
+        elif config["provider"] == "anthropic":
+            # Anthropic doesn't have native JSON mode yet, rely on prompt
+            if config["api_base"]:
+                kwargs["api_base"] = config["api_base"]
+        # Other providers - rely on prompt engineering
+        elif config["api_base"]:
+            kwargs["api_base"] = config["api_base"]
+
+        # Call litellm asynchronously
+        sanitized_key = _sanitize_api_key(config["api_key"]) if config["api_key"] else "(none)"
+        logger.info(
+            f"Calling {config['provider']}/{config['model']} "
+            f"(temp={config['temperature']}, max_tokens={config['max_tokens']}, "
+            f"timeout={config['timeout']}s, key={sanitized_key})"
+        )
+        response = await acompletion(**kwargs)
+
+        content = response.choices[0].message.content
+        return content or ""
+
+    @classmethod
+    async def analyze_image(
+        cls,
+        image_path: str,
+        page_text: str = "",
+    ) -> tuple[AIExtractionResponse, AIExtractionMetadata] | None:
+        """
+        Analyze image and extract price/stock information.
+
+        Args:
+            image_path: Path to screenshot
+            page_text: Optional webpage text context
+
+        Returns:
+            Tuple of (AIExtractionResponse, AIExtractionMetadata) or None on failure
+        """
+        try:
+            # Get AI config
+            loop = asyncio.get_running_loop()
+            config = await loop.run_in_executor(None, cls.get_ai_config)
+
+            logger.info(
+                f"Analyzing image with Provider: {config['provider']}, "
+                f"Model: {config['model']}, Timeout: {config['timeout']}s"
+            )
+
+            # Encode image
+            base64_image = await encode_image(image_path)
+            data_url = f"data:image/jpeg;base64,{base64_image}"
+
+            # Prepare prompt with optional text context
+            cleaned_text = ""
+            if page_text:
+                cleaned_text = clean_text(page_text)
+                if len(cleaned_text) > MAX_TEXT_LENGTH:
+                    cleaned_text = cleaned_text[:MAX_TEXT_LENGTH] + "...(truncated)"
+                logger.info(f"Added text context (original: {len(page_text)}, cleaned: {len(cleaned_text)})")
+
+            prompt = get_extraction_prompt(cleaned_text if cleaned_text else None)
+
+            # Call LLM
+            response_text = await cls.call_llm(prompt, data_url, config)
+
+            logger.info(f"AI Response: {response_text[:200]}...")
+
+            # Parse and validate response
+            repair_used = False
+            try:
+                extraction_result = cls.parse_and_validate_response(response_text)
+            except (ValidationError, json.JSONDecodeError) as e:
+                logger.warning(f"Primary parsing failed: {e}")
+
+                if config["enable_json_repair"]:
+                    try:
+                        extraction_result = await cls.repair_json_response(response_text, config)
+                        repair_used = True
+                        logger.info("JSON repair successful")
+                    except Exception as repair_error:
+                        logger.error(f"JSON repair also failed: {repair_error}")
+                        raise
+                else:
+                    raise
+
+            # Create metadata
+            metadata = AIExtractionMetadata(
+                model_name=config["model"],
+                provider=config["provider"],
+                prompt_version=PROMPT_VERSION,
+                repair_used=repair_used,
+                multi_sample=False,
+                sample_count=1,
+            )
+
+            logger.info(
+                f"Extraction successful: price={extraction_result.price} "
+                f"(conf={extraction_result.price_confidence:.2f}), "
+                f"stock={extraction_result.in_stock} (conf={extraction_result.in_stock_confidence:.2f})"
+            )
+
+            return extraction_result, metadata
+
+        except Exception as e:
+            logger.error(f"Error in analyze_image: {e}", exc_info=True)
+            return None

app/services/item_service.py

@@ -0,0 +1,135 @@
+import logging
+import os
+from datetime import UTC, datetime
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from app import models, schemas
+from app.services.settings_service import SettingsService
+from app.url_validation import URLValidationError, validate_url
+
+logger = logging.getLogger(__name__)
+
+
+class ItemService:
+    @staticmethod
+    def get_items(db: Session):
+        items = db.query(models.Item).all()
+        return [
+            {
+                **item.__dict__,
+                "screenshot_url": f"/screenshots/item_{item.id}.png"
+                if os.path.exists(f"screenshots/item_{item.id}.png")
+                else None,
+            }
+            for item in items
+        ]
+
+    @staticmethod
+    def create_item(db: Session, item: schemas.ItemCreate):
+        logger.info(f"Creating item: {item.name} - {item.url}")
+        try:
+            validate_url(item.url)
+        except URLValidationError as e:
+            raise HTTPException(status_code=400, detail=f"Invalid URL: {e}") from e
+
+        db_item = models.Item(**item.model_dump())
+        db.add(db_item)
+        db.commit()
+        db.refresh(db_item)
+        return db_item
+
+    @staticmethod
+    def update_item(db: Session, item_id: int, item_update: schemas.ItemCreate):
+        db_item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not db_item:
+            raise HTTPException(status_code=404, detail="Item not found")
+
+        for key, value in item_update.model_dump().items():
+            setattr(db_item, key, value)
+
+        db.commit()
+        db.refresh(db_item)
+        return db_item
+
+    @staticmethod
+    def delete_item(db: Session, item_id: int):
+        item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            raise HTTPException(status_code=404, detail="Item not found")
+
+        if os.path.exists(f"screenshots/item_{item_id}.png"):
+            try:
+                os.remove(f"screenshots/item_{item_id}.png")
+            except OSError:
+                pass
+
+        db.delete(item)
+        db.commit()
+        return {"ok": True}
+
+    @staticmethod
+    def get_item(db: Session, item_id: int):
+        return db.query(models.Item).filter(models.Item.id == item_id).first()
+
+    @staticmethod
+    def get_item_data_for_checking(db: Session, item_id: int):
+        item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            return None, None
+
+        settings = {s.key: s.value for s in db.query(models.Settings).all()}
+        profile = item.notification_profile
+
+        item_data = {
+            "id": item.id,
+            "url": item.url,
+            "selector": item.selector,
+            "name": item.name,
+            "current_price": item.current_price,
+            "in_stock": item.in_stock,
+            "target_price": item.target_price,
+            "notification_profile": profile.__dict__ if profile else None,
+        }
+
+        config = {
+            "smart_scroll": settings.get("smart_scroll_enabled", "false").lower() == "true",
+            "smart_scroll_pixels": int(settings.get("smart_scroll_pixels", "350")),
+            "text_context_enabled": settings.get("text_context_enabled", "false").lower() == "true",
+            "text_length": int(settings.get("text_context_length", "5000"))
+            if settings.get("text_context_enabled", "false").lower() == "true"
+            else 0,
+            "scraper_timeout": int(settings.get("scraper_timeout", "90000")),
+        }
+        return item_data, config
+
+    @staticmethod
+    def get_due_items(db: Session):
+        items = db.query(models.Item).filter(models.Item.is_active).all()
+        global_interval = int(SettingsService.get_setting_value(db, "refresh_interval_minutes", "60"))
+        due_items = []
+        now = datetime.now(UTC)
+
+        for item in items:
+            if item.is_refreshing:
+                continue
+
+            interval = (
+                item.notification_profile.check_interval_minutes
+                if item.notification_profile and item.notification_profile.check_interval_minutes
+                else global_interval
+            )
+
+            if not item.last_checked:
+                due_items.append((item.id, interval, -1))
+                continue
+
+            last_checked = (
+                item.last_checked.replace(tzinfo=UTC) if item.last_checked.tzinfo is None else item.last_checked
+            )
+            time_since = (now - last_checked).total_seconds() / 60
+            if time_since >= interval:
+                due_items.append((item.id, interval, int(time_since)))
+
+        return due_items

app/services/notification_service.py

@@ -0,0 +1,89 @@
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from app import models, notification_sender, schemas
+
+
+class NotificationService:
+    @staticmethod
+    def get_notification_profiles(db: Session):
+        return db.query(models.NotificationProfile).all()
+
+    @staticmethod
+    def create_notification_profile(db: Session, profile: schemas.NotificationProfileCreate):
+        db_profile = models.NotificationProfile(**profile.model_dump())
+        db.add(db_profile)
+        db.commit()
+        db.refresh(db_profile)
+        return db_profile
+
+    @staticmethod
+    def delete_notification_profile(db: Session, profile_id: int):
+        profile = db.query(models.NotificationProfile).filter(models.NotificationProfile.id == profile_id).first()
+        if not profile:
+            raise HTTPException(status_code=404, detail="Profile not found")
+        db.delete(profile)
+        db.commit()
+        db.delete(profile)
+        db.commit()
+        return {"ok": True}
+
+    @staticmethod
+    def update_notification_profile(db: Session, profile_id: int, profile_data: schemas.NotificationProfileUpdate):
+        profile = db.query(models.NotificationProfile).filter(models.NotificationProfile.id == profile_id).first()
+        if not profile:
+            raise HTTPException(status_code=404, detail="Profile not found")
+
+        for key, value in profile_data.model_dump().items():
+            setattr(profile, key, value)
+
+        db.commit()
+        db.refresh(profile)
+        return profile
+
+    @staticmethod
+    async def send_item_notifications(
+        item_data: dict,
+        price: float | None,
+        old_price: float | None,
+        in_stock: bool | None,
+        old_stock: bool | None,
+    ):
+        profile = item_data["notification_profile"]
+        if not profile:
+            return
+
+        if profile["notify_on_price_drop"] and price is not None and old_price is not None:
+            if price < old_price:
+                drop_percent = ((old_price - price) / old_price) * 100
+                if drop_percent >= profile["price_drop_threshold_percent"]:
+                    await notification_sender.send_notification(
+                        [profile["apprise_url"]],
+                        f"Price Drop Alert: {item_data['name']}",
+                        f"Price dropped by {drop_percent:.1f}%! Now ${price} (was ${old_price})",
+                    )
+
+        if (
+            profile["notify_on_target_price"]
+            and price is not None
+            and item_data["target_price"]
+            and price <= item_data["target_price"]
+        ):
+            await notification_sender.send_notification(
+                [profile["apprise_url"]],
+                f"Target Price Alert: {item_data['name']}",
+                f"Price is ${price} (Target: ${item_data['target_price']})",
+            )
+
+        if (
+            profile["notify_on_stock_change"]
+            and in_stock is not None
+            and old_stock is not None
+            and in_stock != old_stock
+        ):
+            status = "In Stock" if in_stock else "Out of Stock"
+            await notification_sender.send_notification(
+                [profile["apprise_url"]],
+                f"Stock Alert: {item_data['name']}",
+                f"Item is now {status}",
+            )

app/services/scheduler_service.py

@@ -0,0 +1,147 @@
+import asyncio
+import logging
+from datetime import UTC, datetime
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from app import database, models
+from app.ai_schema import AIExtractionMetadata, AIExtractionResponse
+from app.services.ai_service import AIService
+from app.services.item_service import ItemService
+from app.services.notification_service import NotificationService
+from app.services.scraper_service import ScraperService
+
+logger = logging.getLogger(__name__)
+scheduler = AsyncIOScheduler()
+
+PRICE_CHANGE_THRESHOLD_PERCENT = 20.0
+LOW_CONFIDENCE_THRESHOLD = 0.7
+
+
+def _get_thresholds():
+    with database.SessionLocal() as session:
+        settings = {s.key: s.value for s in session.query(models.Settings).all()}
+        return {
+            "price": float(settings.get("confidence_threshold_price", "0.5")),
+            "stock": float(settings.get("confidence_threshold_stock", "0.5")),
+        }
+
+
+def _update_db_result(
+    item_id: int,
+    extraction: AIExtractionResponse,
+    metadata: AIExtractionMetadata,
+    thresholds: dict,
+    screenshot_path: str,
+):
+    with database.SessionLocal() as session:
+        item = session.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            return None, None
+
+        old_price, old_stock = item.current_price, item.in_stock
+        price, in_stock = extraction.price, extraction.in_stock
+        p_conf, s_conf = extraction.price_confidence, extraction.in_stock_confidence
+
+        if price is not None:
+            if p_conf >= thresholds["price"]:
+                if (
+                    old_price
+                    and (abs(price - old_price) / old_price * 100 > PRICE_CHANGE_THRESHOLD_PERCENT)
+                    and p_conf < LOW_CONFIDENCE_THRESHOLD
+                ):
+                    item.last_error = f"Uncertain: Large price change with low confidence ({p_conf:.2f})"
+                else:
+                    item.last_error = None
+                item.current_price = price
+                item.current_price_confidence = p_conf
+
+            session.add(
+                models.PriceHistory(
+                    item_id=item.id,
+                    price=price,
+                    screenshot_path=screenshot_path,
+                    price_confidence=p_conf,
+                    in_stock_confidence=s_conf,
+                    ai_model=metadata.model_name,
+                    ai_provider=metadata.provider,
+                    prompt_version=metadata.prompt_version,
+                    repair_used=metadata.repair_used,
+                )
+            )
+
+        if in_stock is not None and s_conf >= thresholds["stock"]:
+            item.in_stock = in_stock
+            item.in_stock_confidence = s_conf
+
+        item.last_checked = datetime.now(UTC)
+        item.is_refreshing = False
+        if item.last_error and not item.last_error.startswith("Uncertain:"):
+            item.last_error = None
+
+        session.commit()
+        return old_price, old_stock
+
+
+def _update_db_error(item_id, error_msg):
+    with database.SessionLocal() as session:
+        if item := session.query(models.Item).filter(models.Item.id == item_id).first():
+            item.is_refreshing = False
+            item.last_error = error_msg
+            item.last_checked = datetime.now(UTC)
+            session.commit()
+
+
+async def process_item_check(item_id: int):
+    loop = asyncio.get_running_loop()
+    with database.SessionLocal() as session:
+        item_data, config = await loop.run_in_executor(None, ItemService.get_item_data_for_checking, session, item_id)
+
+    if not item_data:
+        logger.error(f"process_item_check: Item ID {item_id} not found")
+        return
+
+    try:
+        logger.info(f"Checking item: {item_data['name']} ({item_data['url']})")
+        screenshot_path, page_text = await ScraperService.scrape_item(
+            item_data["url"],
+            item_data["selector"],
+            item_id,
+            smart_scroll=config["smart_scroll"],
+            scroll_pixels=config["smart_scroll_pixels"],
+            text_length=config["text_length"],
+            timeout=config["scraper_timeout"],
+        )
+
+        if not screenshot_path:
+            raise Exception("Failed to capture screenshot")
+
+        if not (ai_result := await AIService.analyze_image(screenshot_path, page_text=page_text)):
+            raise Exception("AI analysis failed")
+
+        extraction, metadata = ai_result
+        thresholds = await loop.run_in_executor(None, _get_thresholds)
+        old_price, old_stock = await loop.run_in_executor(
+            None, _update_db_result, item_id, extraction, metadata, thresholds, screenshot_path
+        )
+
+        await NotificationService.send_item_notifications(
+            item_data, extraction.price, old_price, extraction.in_stock, old_stock
+        )
+
+    except Exception as e:
+        logger.error(f"Error in process_item_check: {e}")
+        await loop.run_in_executor(None, _update_db_error, item_id, str(e))
+
+
+async def scheduled_refresh():
+    logger.info("Heartbeat: Checking for items due for refresh")
+    loop = asyncio.get_running_loop()
+    try:
+        with database.SessionLocal() as session:
+            due_items = await loop.run_in_executor(None, ItemService.get_due_items, session)
+
+        for item_id, _, _ in due_items:
+            await process_item_check(item_id)
+    except Exception as e:
+        logger.error(f"Error in scheduled refresh: {e}", exc_info=True)

app/services/scraper_service.py

@@ -0,0 +1,185 @@
+import asyncio
+import logging
+import os
+
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright.async_api import async_playwright
+
+logger = logging.getLogger(__name__)
+
+BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
+
+
+class ScraperService:
+    @staticmethod
+    async def scrape_item(  # noqa: PLR0913, PLR0912, PLR0915
+        url: str,
+        selector: str | None = None,
+        item_id: int | None = None,
+        smart_scroll: bool = False,
+        scroll_pixels: int = 350,
+        text_length: int = 0,
+        timeout: int = 90000,
+    ) -> tuple[str | None, str]:
+        """
+        Scrapes the given URL using Browserless and Playwright.
+        Returns a tuple: (screenshot_path, page_text)
+
+        Args:
+            url: Target URL to scrape
+            selector: Optional CSS selector to focus on
+            item_id: Optional item ID for screenshot naming
+            smart_scroll: Enable scrolling to load lazy content
+            scroll_pixels: Number of pixels to scroll (must be positive)
+            text_length: Number of characters to extract (0 = disabled)
+            timeout: Page load timeout in milliseconds
+        """
+        # Input validation
+        if scroll_pixels <= 0:
+            logger.warning(f"Invalid scroll_pixels value: {scroll_pixels}, using default 350")
+            scroll_pixels = 350
+
+        if timeout <= 0:
+            logger.warning(f"Invalid timeout value: {timeout}, using default 90000")
+            timeout = 90000
+        async with async_playwright() as p:
+            browser = None
+            try:
+                logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+                browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
+                context = await browser.new_context(
+                    viewport={"width": 1920, "height": 1080},
+                    user_agent=(
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/120.0.0.0 Safari/537.36"
+                    ),
+                )
+
+                # Stealth mode / Ad blocking attempts
+                await context.route("**/*", lambda route: route.continue_())
+
+                page = await context.new_page()
+
+                logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
+                try:
+                    # First wait for domcontentloaded - this is the minimum we need
+                    await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
+                    logger.info(f"Page loaded (domcontentloaded): {url}")
+
+                    # Then try to wait for networkidle, but don't fail if it times out
+                    # This helps with heavy pages that never fully settle
+                    try:
+                        await page.wait_for_load_state("networkidle", timeout=5000)
+                        logger.info("Network idle reached")
+                    except PlaywrightTimeoutError:
+                        logger.info("Network idle timed out (non-critical), proceeding...")
+
+                except Exception as e:
+                    logger.error(f"Error navigating to {url}: {e}")
+                    # Try to take screenshot anyway if page partially loaded
+                    pass
+
+                # Wait a bit for dynamic content if needed
+                await page.wait_for_timeout(2000)
+
+                # Try to close common popups
+                logger.info("Attempting to close popups...")
+                popup_selectors = [
+                    "button[aria-label='Close']",
+                    "button[aria-label='close']",
+                    ".close-button",
+                    ".modal-close",
+                    "svg[data-name='Close']",
+                    "[class*='popup'] button",
+                    "[class*='modal'] button",
+                    "button:has-text('No, thanks')",
+                    "button:has-text('No thanks')",
+                    "a:has-text('No, thanks')",
+                    "div[role='dialog'] button[aria-label='Close']",
+                ]
+
+                for popup_selector in popup_selectors:
+                    try:
+                        if await page.locator(popup_selector).count() > 0:
+                            logger.info(f"Found popup close button: {popup_selector}")
+                            # Try to click it. If it fails, catch and continue
+                            await page.locator(popup_selector).first.click(timeout=2000)
+                            await page.wait_for_timeout(1000)  # Wait for animation
+                    except Exception as e:
+                        logger.debug(f"Could not close popup with selector {popup_selector}: {e}")
+
+                # Also try pressing Escape
+                try:
+                    await page.keyboard.press("Escape")
+                except Exception as e:
+                    logger.debug(f"Could not press Escape key: {e}")
+
+                if selector:
+                    try:
+                        logger.info(f"Waiting for selector: {selector}")
+                        await page.wait_for_selector(selector, timeout=5000)
+                        # Scroll to element
+                        element = page.locator(selector).first
+                        await element.scroll_into_view_if_needed()
+                        logger.info(f"Scrolled to selector: {selector}")
+                    except Exception as e:
+                        logger.warning(f"Selector {selector} not found or timed out: {e}")
+                else:
+                    # Auto-detect price if no selector
+                    logger.info("No selector provided. Attempting to find price element...")
+                    try:
+                        # Look for common price patterns
+                        price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/")
+                        if await price_locator.count() > 0:
+                            # Pick the first one that looks visible and reasonable size
+                            # This is heuristic
+                            await price_locator.first.scroll_into_view_if_needed()
+                            logger.info("Scrolled to potential price element")
+                    except Exception as e:
+                        logger.warning(f"Auto-price detection failed: {e}")
+
+                # Smart Scroll
+                if smart_scroll:
+                    logger.info(f"Performing smart scroll ({scroll_pixels}px)...")
+                    try:
+                        await page.evaluate(f"window.scrollBy(0, {scroll_pixels})")
+                        await page.wait_for_timeout(1000)
+                    except Exception as e:
+                        logger.warning(f"Smart scroll failed: {e}")
+
+                # Text Extraction
+                page_text = ""
+                if text_length > 0:
+                    try:
+                        logger.info(f"Extracting text (limit: {text_length} chars)...")
+                        # Get text from body
+                        raw_text = await page.inner_text("body")
+                        # Simple truncation
+                        page_text = raw_text[:text_length]
+                        logger.info(f"Extracted {len(page_text)} characters")
+                    except Exception as e:
+                        logger.error(f"Text extraction failed: {e}")
+
+                # Take screenshot
+                screenshot_dir = "screenshots"
+                os.makedirs(screenshot_dir, exist_ok=True)
+                if item_id:
+                    filename = f"{screenshot_dir}/item_{item_id}.png"
+                else:
+                    url_part = url.split("//")[-1].replace("/", "_")
+                    timestamp = asyncio.get_event_loop().time()
+                    filename = f"{screenshot_dir}/{url_part}_{timestamp}.png"
+
+                await page.screenshot(path=filename, full_page=False)
+                logger.info(f"Screenshot saved to {filename}")
+
+                return filename, page_text
+
+            except Exception as e:
+                logger.error(f"Error scraping {url}: {e}")
+                return None, ""
+
+            finally:
+                if browser:
+                    await browser.close()

app/services/settings_service.py

@@ -0,0 +1,25 @@
+from sqlalchemy.orm import Session
+
+from app import models, schemas
+
+
+class SettingsService:
+    @staticmethod
+    def get_settings(db: Session):
+        return db.query(models.Settings).all()
+
+    @staticmethod
+    def update_setting(db: Session, setting: schemas.SettingsUpdate):
+        db_setting = db.query(models.Settings).filter(models.Settings.key == setting.key).first()
+        if db_setting:
+            db_setting.value = setting.value
+        else:
+            db_setting = models.Settings(key=setting.key, value=setting.value)
+            db.add(db_setting)
+        db.commit()
+        return db_setting
+
+    @staticmethod
+    def get_setting_value(db: Session, key: str, default: str | None = None):
+        setting = db.query(models.Settings).filter(models.Settings.key == key).first()
+        return setting.value if setting else default