Add files via upload

app/ai_schema.py

@@ -0,0 +1,202 @@
+"""
+Unified JSON schema for AI extraction responses.
+
+This module defines the canonical schema for all AI model responses,
+including confidence scores and metadata tracking.
+"""
+
+import re
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+from app.utils.text import filter_relevant_text
+
+# Schema version for tracking prompt/schema changes
+PROMPT_VERSION = "v2.0"
+
+# Default confidence thresholds
+DEFAULT_PRICE_CONFIDENCE_THRESHOLD = 0.5
+DEFAULT_STOCK_CONFIDENCE_THRESHOLD = 0.5
+DEFAULT_MULTI_SAMPLE_THRESHOLD = 0.6
+
+# Text filtering constants
+MIN_SNIPPET_LENGTH = 10
+SNIPPET_MERGE_DISTANCE = 50
+SNIPPET_CONTEXT_WINDOW = 100
+
+
+class AIExtractionResponse(BaseModel):
+    """
+    Canonical schema for AI extraction responses.
+
+    All AI models must return data matching this schema.
+    """
+
+    price: float | None = Field(
+        None,
+        description="Extracted price as a number, or null if no price found",
+    )
+    currency: str = Field(
+        "USD",
+        description="Currency code (ISO 4217)",
+    )
+    in_stock: bool | None = Field(
+        None,
+        description="Stock status: true if in stock, false if out of stock, null if unclear",
+    )
+    price_confidence: float = Field(
+        0.0,
+        ge=0.0,
+        le=1.0,
+        description="Confidence in price extraction (0.0 to 1.0)",
+    )
+    in_stock_confidence: float = Field(
+        0.0,
+        ge=0.0,
+        le=1.0,
+        description="Confidence in stock status extraction (0.0 to 1.0)",
+    )
+    source_type: Literal["image", "text", "both"] = Field(
+        "image",
+        description="Source of extraction: image, text, or both",
+    )
+
+    @field_validator("price_confidence", "in_stock_confidence", mode="before")
+    @classmethod
+    def clamp_confidence(cls, v):
+        """Clamp confidence values to [0.0, 1.0] range."""
+        if v is None:
+            return 0.0
+        return max(0.0, min(1.0, float(v)))
+
+    @field_validator("price", mode="before")
+    @classmethod
+    def normalize_price(cls, v):
+        """Normalize price to float or None."""
+        if v is None or v in ("null", ""):
+            return None
+        if isinstance(v, str):
+            # Remove currency symbols and commas
+            cleaned = re.sub(r"[^\d.]", "", v)
+            if cleaned:
+                return float(cleaned)
+            return None
+        return float(v)
+
+    @field_validator("in_stock", mode="before")
+    @classmethod
+    def normalize_stock(cls, v):
+        """Normalize stock status to boolean or None."""
+        if v is None or v == "null":
+            return None
+        if isinstance(v, str):
+            v_lower = v.lower().strip()
+            if v_lower in ("true", "yes", "in stock", "available", "1"):
+                return True
+            if v_lower in ("false", "no", "out of stock", "unavailable", "0"):
+                return False
+            return None
+        return bool(v)
+
+
+class AIExtractionMetadata(BaseModel):
+    """
+    Metadata about an AI extraction operation.
+
+    Tracks which model was used, whether repair was needed, etc.
+    """
+
+    model_name: str = Field(..., description="AI model identifier (e.g., 'gpt-4o')")
+    provider: str = Field(..., description="AI provider (e.g., 'openai', 'ollama')")
+    prompt_version: str = Field(PROMPT_VERSION, description="Version of the extraction prompt used")
+    repair_used: bool = Field(False, description="Whether JSON repair fallback was used")
+    multi_sample: bool = Field(False, description="Whether multi-sample validation was used")
+    sample_count: int = Field(1, description="Number of samples generated (for multi-sample)")
+
+
+# Prompt template for schema-first extraction
+EXTRACTION_PROMPT_TEMPLATE = """Extract product price and stock status from the image.
+
+**PRICE:**
+- Find the main current price
+- Ignore crossed-out prices
+- Extract number only (no symbols)
+- If unclear: set null and confidence < 0.5
+
+**STOCK:**
+- TRUE if: "Add to Cart", "Buy Now", "In Stock", "Available"
+- FALSE if: "Out of Stock", "Sold Out", "Unavailable", "Notify Me"
+- NULL if unclear or not shown
+
+**CONFIDENCE (0.0 to 1.0):**
+- 0.9-1.0: Very certain
+- 0.5-0.8: Moderately certain
+- Below 0.5: Unsure
+
+Respond ONLY with valid JSON:
+{{
+  "price": <number or null>,
+  "currency": "USD",
+  "in_stock": <true, false, or null>,
+  "price_confidence": <0.0 to 1.0>,
+  "in_stock_confidence": <0.0 to 1.0>,
+  "source_type": "both"
+}}
+
+{context_section}"""
+
+# Repair prompt template
+REPAIR_PROMPT_TEMPLATE = """Convert the following text into valid JSON matching this schema:
+
+{{
+  "price": <number or null>,
+  "currency": "<ISO currency code, default USD>",
+  "in_stock": <true, false, or null>,
+  "price_confidence": <number from 0.0 to 1.0>,
+  "in_stock_confidence": <number from 0.0 to 1.0>,
+  "source_type": "<image, text, or both>"
+}}
+
+Rules:
+- Extract numeric price value only (no symbols)
+- Boolean values must be true, false, or null (not strings)
+- Confidence values must be numbers between 0.0 and 1.0
+- Respond with ONLY the JSON object, no other text
+
+Text to convert:
+{raw_output}"""
+
+
+def get_extraction_prompt(page_text: str | None = None) -> str:
+    """
+    Generate the extraction prompt with optional text context.
+
+    Args:
+        page_text: Optional webpage text to include as context
+
+    Returns:
+        Formatted prompt string
+    """
+    if page_text:
+        # Apply smart filtering to extract only relevant snippets
+        filtered_text = filter_relevant_text(page_text, max_length=1500)
+        context_section = f"""**Relevant text from page:**
+{filtered_text}"""
+    else:
+        context_section = ""
+
+    return EXTRACTION_PROMPT_TEMPLATE.format(context_section=context_section)
+
+
+def get_repair_prompt(raw_output: str) -> str:
+    """
+    Generate the repair prompt for fixing invalid JSON.
+
+    Args:
+        raw_output: Raw AI output that failed parsing
+
+    Returns:
+        Formatted repair prompt
+    """
+    return REPAIR_PROMPT_TEMPLATE.format(raw_output=raw_output[:1000])

app/alembic/env.py

@@ -0,0 +1,82 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+
+from alembic import context
+from app.database import DATABASE_URL
+from app.models import Base
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+
+
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = DATABASE_URL
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    configuration = config.get_section(config.config_ini_section)
+    configuration["sqlalchemy.url"] = DATABASE_URL
+
+    connectable = engine_from_config(
+        configuration,
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

app/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

app/alembic/versions/6ecb6f39fd8c_add_refresh_fields.py

@@ -0,0 +1,37 @@
+"""Add refresh fields
+
+Revision ID: 6ecb6f39fd8c
+Revises: initial_migration
+Create Date: 2025-11-19 12:53:09.412700
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+from app.models import Base
+
+target_metadata = Base.metadata
+
+
+# revision identifiers, used by Alembic.
+revision: str = "6ecb6f39fd8c"
+down_revision: str | None = "initial_migration"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("items", sa.Column("is_refreshing", sa.Boolean(), nullable=True))
+    op.add_column("items", sa.Column("last_error", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("items", "last_error")
+    op.drop_column("items", "is_refreshing")
+    # ### end Alembic commands ###

app/alembic/versions/7f8e9d2a1b3c_add_performance_indexes.py

@@ -0,0 +1,39 @@
+"""add_performance_indexes
+
+Revision ID: 7f8e9d2a1b3c
+Revises: a1b2c3d4e5f6
+Create Date: 2025-11-19 19:14:00.000000
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7f8e9d2a1b3c"
+down_revision = "a1b2c3d4e5f6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add indexes for performance optimization
+    # Items table indexes
+    op.create_index("ix_items_last_checked", "items", ["last_checked"])
+    op.create_index("ix_items_is_active", "items", ["is_active"])
+    # Note: url already has an index from the model definition
+
+    # Composite index for common query pattern (active items due for refresh)
+    op.create_index("ix_items_active_last_checked", "items", ["is_active", "last_checked"])
+
+    # Price history indexes for time-series queries
+    op.create_index("ix_price_history_timestamp", "price_history", ["timestamp"])
+    op.create_index("ix_price_history_item_timestamp", "price_history", ["item_id", "timestamp"])
+
+
+def downgrade():
+    # Remove indexes in reverse order
+    op.drop_index("ix_price_history_item_timestamp", table_name="price_history")
+    op.drop_index("ix_price_history_timestamp", table_name="price_history")
+    op.drop_index("ix_items_active_last_checked", table_name="items")
+    op.drop_index("ix_items_is_active", table_name="items")
+    op.drop_index("ix_items_last_checked", table_name="items")

app/alembic/versions/a1b2c3d4e5f6_add_confidence_metadata.py

@@ -0,0 +1,55 @@
+"""Add confidence and AI metadata
+
+Revision ID: a1b2c3d4e5f6
+Revises: 6ecb6f39fd8c
+Create Date: 2025-11-19 13:55:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+from app.models import Base
+
+target_metadata = Base.metadata
+
+
+# revision identifiers, used by Alembic.
+revision: str = "a1b2c3d4e5f6"
+down_revision: str | None = "6ecb6f39fd8c"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add confidence scores and AI metadata to price_history and items tables."""
+
+    # Add columns to price_history table
+    op.add_column("price_history", sa.Column("price_confidence", sa.Float(), nullable=True))
+    op.add_column("price_history", sa.Column("in_stock_confidence", sa.Float(), nullable=True))
+    op.add_column("price_history", sa.Column("ai_model", sa.String(), nullable=True))
+    op.add_column("price_history", sa.Column("ai_provider", sa.String(), nullable=True))
+    op.add_column("price_history", sa.Column("prompt_version", sa.String(), nullable=True))
+    op.add_column("price_history", sa.Column("repair_used", sa.Boolean(), nullable=True, server_default="false"))
+
+    # Add columns to items table for quick access to latest confidence
+    op.add_column("items", sa.Column("current_price_confidence", sa.Float(), nullable=True))
+    op.add_column("items", sa.Column("in_stock_confidence", sa.Float(), nullable=True))
+
+
+def downgrade() -> None:
+    """Remove confidence scores and AI metadata columns."""
+
+    # Remove columns from items table
+    op.drop_column("items", "in_stock_confidence")
+    op.drop_column("items", "current_price_confidence")
+
+    # Remove columns from price_history table
+    op.drop_column("price_history", "repair_used")
+    op.drop_column("price_history", "prompt_version")
+    op.drop_column("price_history", "ai_provider")
+    op.drop_column("price_history", "ai_model")
+    op.drop_column("price_history", "in_stock_confidence")
+    op.drop_column("price_history", "price_confidence")

app/alembic/versions/initial_migration.py

@@ -0,0 +1,116 @@
+"""Initial migration
+
+Revision ID: initial_migration
+Revises:
+Create Date: 2023-11-19 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "initial_migration"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # We are assuming the database is already created by the previous code
+    # But for a fresh install, we need to create tables.
+    # However, since we are retrofitting, we should use --autogenerate usually.
+    # But I cannot run that easily.
+    # So I will define the tables here to be safe.
+
+    # Check if tables exist before creating to avoid errors on existing DBs
+    # Actually, standard alembic practice is to define them.
+    # But since we had `Base.metadata.create_all`, tables might exist.
+    # Ideally we would use `alembic stamp head` on existing deployments.
+
+    # For now, I will leave this empty and assume the user will run
+    # `alembic revision --autogenerate` if they want to capture the state,
+    # or I should try to capture it.
+    # Given I can't run the DB, I'll leave it as a placeholder or try to define it.
+    # Let's define it properly so new installs work.
+
+    # ... actually, writing all create_table calls manually is error prone without running it.
+    # I will rely on the fact that I removed `create_all` from main.py, so this MUST do it.
+
+    op.create_table(
+        "notification_profiles",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("apprise_url", sa.String(), nullable=True),
+        sa.Column("notify_on_price_drop", sa.Boolean(), nullable=True),
+        sa.Column("notify_on_target_price", sa.Boolean(), nullable=True),
+        sa.Column("price_drop_threshold_percent", sa.Float(), nullable=True),
+        sa.Column("notify_on_stock_change", sa.Boolean(), nullable=True),
+        sa.Column("check_interval_minutes", sa.Integer(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_notification_profiles_id"), "notification_profiles", ["id"], unique=False)
+    op.create_index(op.f("ix_notification_profiles_name"), "notification_profiles", ["name"], unique=True)
+
+    op.create_table(
+        "items",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("url", sa.String(), nullable=True),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("selector", sa.String(), nullable=True),
+        sa.Column("target_price", sa.Float(), nullable=True),
+        sa.Column("check_interval_minutes", sa.Integer(), nullable=True),
+        sa.Column("current_price", sa.Float(), nullable=True),
+        sa.Column("in_stock", sa.Boolean(), nullable=True),
+        sa.Column("tags", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("is_active", sa.Boolean(), nullable=True),
+        sa.Column("last_checked", sa.DateTime(), nullable=True),
+        sa.Column("notification_profile_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["notification_profile_id"],
+            ["notification_profiles.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_items_id"), "items", ["id"], unique=False)
+    op.create_index(op.f("ix_items_url"), "items", ["url"], unique=False)
+
+    op.create_table(
+        "price_history",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("item_id", sa.Integer(), nullable=True),
+        sa.Column("price", sa.Float(), nullable=True),
+        sa.Column("timestamp", sa.DateTime(), nullable=True),
+        sa.Column("screenshot_path", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["item_id"],
+            ["items.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_price_history_id"), "price_history", ["id"], unique=False)
+
+    op.create_table(
+        "settings",
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("value", sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint("key"),
+    )
+    op.create_index(op.f("ix_settings_key"), "settings", ["key"], unique=False)
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_settings_key"), table_name="settings")
+    op.drop_table("settings")
+    op.drop_index(op.f("ix_price_history_id"), table_name="price_history")
+    op.drop_table("price_history")
+    op.drop_index(op.f("ix_items_url"), table_name="items")
+    op.drop_index(op.f("ix_items_id"), table_name="items")
+    op.drop_table("items")
+    op.drop_index(op.f("ix_notification_profiles_name"), table_name="notification_profiles")
+    op.drop_index(op.f("ix_notification_profiles_id"), table_name="notification_profiles")
+    op.drop_table("notification_profiles")

app/alembic/versions/performance_indexes_v2.py

@@ -0,0 +1,46 @@
+"""Add performance indexes
+
+Revision ID: performance_indexes_v2
+Revises: 7f8e9d2a1b3c
+Create Date: 2025-11-19
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "performance_indexes_v2"
+down_revision = "7f8e9d2a1b3c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add indexes only if they don't exist
+    # Using raw SQL with IF NOT EXISTS for safety
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_price_history_timestamp
+        ON price_history (timestamp)
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_price_history_item_timestamp
+        ON price_history (item_id, timestamp)
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_items_is_active
+        ON items (is_active)
+        """
+    )
+
+
+def downgrade():
+    op.drop_index("ix_items_is_active", table_name="items", if_exists=True)
+    op.drop_index("ix_price_history_item_timestamp", table_name="price_history", if_exists=True)
+    op.drop_index("ix_price_history_timestamp", table_name="price_history", if_exists=True)

app/database.py

@@ -0,0 +1,58 @@
+import logging
+import os
+
+from sqlalchemy import create_engine, event
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import QueuePool
+
+logger = logging.getLogger(__name__)
+
+# Remove hardcoded credentials - require environment variable
+DATABASE_URL = os.getenv("DATABASE_URL")
+
+if not DATABASE_URL:
+    raise ValueError(
+        "DATABASE_URL environment variable is required. Example: postgresql://user:password@localhost:5432/pricewatch"
+    )
+
+# Connection pool configuration
+#  pool_size: number of connections to maintain
+#  max_overflow: max number of connections above pool_size
+#  pool_pre_ping: verify connections before using (prevents stale connections)
+#  pool_recycle: recycle connections after N seconds (prevents timeout issues)
+engine = create_engine(
+    DATABASE_URL,
+    poolclass=QueuePool,
+    pool_size=5,
+    max_overflow=10,
+    pool_pre_ping=True,
+    pool_recycle=3600,  # 1 hour
+    echo=os.getenv("SQL_ECHO", "false").lower() == "true",
+)
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+
+# Log database connection events for debugging
+@event.listens_for(engine, "connect")
+def receive_connect(dbapi_conn, connection_record):
+    logger.debug("Database connection established")
+
+
+@event.listens_for(engine, "close")
+def receive_close(dbapi_conn, connection_record):
+    logger.debug("Database connection closed")
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    except Exception:
+        db.rollback()
+        raise
+    finally:
+        db.close()

app/limiter.py

@@ -0,0 +1,4 @@
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+
+limiter = Limiter(key_func=get_remote_address)

app/main.py

@@ -0,0 +1,89 @@
+import logging
+import os
+from contextlib import asynccontextmanager
+
+from apscheduler.triggers.interval import IntervalTrigger
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from slowapi import _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+
+from app.limiter import limiter
+from app.routers import items, jobs, notifications, settings
+from app.services.scheduler_service import scheduled_refresh, scheduler
+
+# Configure logging
+logging.basicConfig(
+    level=os.getenv("LOG_LEVEL", "INFO").upper(),
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logger.info("Starting smart scheduler (Heartbeat: 1 minute)")
+    scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=1), id="refresh_job", replace_existing=True)
+    scheduler.start()
+    logger.info("Application started")
+    yield
+    logger.info("Shutting down scheduler...")
+    scheduler.shutdown(wait=True)
+    logger.info("Application shutdown complete")
+
+
+app = FastAPI(title="Pricecious API", version="0.1.0", lifespan=lifespan)
+
+# Rate Limiting & CORS
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)  # type: ignore
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=os.getenv("CORS_ORIGINS", "*").split(","),
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.middleware("http")
+async def log_requests(request: Request, call_next):
+    logger.debug(f"Request: {request.method} {request.url}")
+    response = await call_next(request)
+    logger.debug(f"Response: {response.status_code}")
+    return response
+
+
+# Static Files
+if os.path.exists("static"):
+    app.mount("/assets", StaticFiles(directory="static/assets"), name="assets")
+os.makedirs("screenshots", exist_ok=True)
+app.mount("/screenshots", StaticFiles(directory="screenshots"), name="screenshots")
+
+# Routers
+for router in [notifications.router, items.router, settings.router, jobs.router]:
+    app.include_router(router, prefix="/api")
+
+
+@app.get("/api/")
+def read_root():
+    return {"message": "Welcome to Pricecious API"}
+
+
+# Frontend Serving
+@app.get("/{full_path:path}")
+async def serve_spa(full_path: str):
+    if full_path.startswith(("api", "screenshots", "assets")):
+        return {"message": "Not found"}
+
+    # Serve static files from root (favicon, logo, etc.)
+    static_file_path = f"static/{full_path}"
+    if os.path.isfile(static_file_path):
+        return FileResponse(static_file_path)
+
+    # Default to SPA index.html
+    if os.path.exists("static/index.html"):
+        return FileResponse("static/index.html")
+    return {"message": "Frontend not built or not found"}

app/models.py

@@ -0,0 +1,79 @@
+from datetime import UTC, datetime
+
+from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import relationship
+
+from .database import Base
+
+
+class NotificationProfile(Base):
+    __tablename__ = "notification_profiles"
+
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, unique=True, index=True)
+    apprise_url = Column(String)
+    notify_on_price_drop = Column(Boolean, default=True)
+    notify_on_target_price = Column(Boolean, default=True)
+    price_drop_threshold_percent = Column(Float, default=10.0)
+    notify_on_stock_change = Column(Boolean, default=True)
+    check_interval_minutes = Column(Integer, default=60)
+
+    items = relationship("Item", back_populates="notification_profile")
+
+
+class Item(Base):
+    __tablename__ = "items"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    url: str = Column(String, index=True)  # type: ignore
+    name: str = Column(String)  # type: ignore
+    selector: str | None = Column(String, nullable=True)  # type: ignore
+    target_price: float | None = Column(Float, nullable=True)  # type: ignore
+    check_interval_minutes: int = Column(Integer, default=60)  # type: ignore
+
+    # New fields
+    current_price: float | None = Column(Float, nullable=True)  # type: ignore
+    in_stock: bool | None = Column(Boolean, nullable=True)  # type: ignore
+    tags: str | None = Column(String, nullable=True)  # type: ignore
+    description: str | None = Column(String, nullable=True)  # type: ignore
+
+    # Confidence scores for latest extraction
+    current_price_confidence: float | None = Column(Float, nullable=True)  # type: ignore
+    in_stock_confidence: float | None = Column(Float, nullable=True)  # type: ignore
+
+    is_active: bool = Column(Boolean, default=True)  # type: ignore
+    last_checked: datetime | None = Column(DateTime, nullable=True)  # type: ignore
+    is_refreshing: bool = Column(Boolean, default=False)  # type: ignore
+    last_error: str | None = Column(String, nullable=True)  # type: ignore
+
+    notification_profile_id: int | None = Column(Integer, ForeignKey("notification_profiles.id"), nullable=True)  # type: ignore
+    notification_profile = relationship("NotificationProfile", back_populates="items")
+
+    price_history = relationship("PriceHistory", back_populates="item", cascade="all, delete-orphan")
+
+
+class PriceHistory(Base):
+    __tablename__ = "price_history"
+
+    id: int = Column(Integer, primary_key=True, index=True)  # type: ignore
+    item_id: int = Column(Integer, ForeignKey("items.id"))  # type: ignore
+    price: float = Column(Float)  # type: ignore
+    timestamp: datetime = Column(DateTime, default=lambda: datetime.now(UTC))  # type: ignore
+    screenshot_path: str | None = Column(String, nullable=True)  # type: ignore
+
+    # Confidence scores and AI metadata
+    price_confidence: float | None = Column(Float, nullable=True)  # type: ignore
+    in_stock_confidence: float | None = Column(Float, nullable=True)  # type: ignore
+    ai_model: str | None = Column(String, nullable=True)  # type: ignore
+    ai_provider: str | None = Column(String, nullable=True)  # type: ignore
+    prompt_version: str | None = Column(String, nullable=True)  # type: ignore
+    repair_used: bool | None = Column(Boolean, nullable=True, default=False)  # type: ignore
+
+    item = relationship("Item", back_populates="price_history")
+
+
+class Settings(Base):
+    __tablename__ = "settings"
+
+    key = Column(String, primary_key=True, index=True)
+    value = Column(Text)

app/notification_sender.py

@@ -0,0 +1,36 @@
+import asyncio
+import logging
+
+import apprise
+
+logger = logging.getLogger(__name__)
+
+
+def _send_sync(urls: list, title: str, body: str):
+    """
+    Synchronous notification sending.
+    """
+    if not urls:
+        return
+
+    apobj = apprise.Apprise()
+
+    for url in urls:
+        apobj.add(url)
+
+    try:
+        apobj.notify(
+            body=body,
+            title=title,
+        )
+        logger.info(f"Notification sent: {title}")
+    except Exception as e:
+        logger.error(f"Error sending notification: {e}")
+
+
+async def send_notification(urls: list, title: str, body: str):
+    """
+    Async wrapper for sending notifications.
+    """
+    loop = asyncio.get_running_loop()
+    await loop.run_in_executor(None, _send_sync, urls, title, body)

app/routers/items.py

@@ -0,0 +1,46 @@
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
+from sqlalchemy.orm import Session
+
+from app import database, schemas
+from app.limiter import limiter
+from app.services.item_service import ItemService
+from app.services.scheduler_service import process_item_check
+
+router = APIRouter(prefix="/items", tags=["items"])
+
+
+@router.get("", response_model=list[schemas.ItemResponse])
+def get_items(db: Session = Depends(database.get_db)):
+    return ItemService.get_items(db)
+
+
+@router.post("", response_model=schemas.ItemResponse)
+def create_item(item: schemas.ItemCreate, db: Session = Depends(database.get_db)):
+    return ItemService.create_item(db, item)
+
+
+@router.put("/{item_id}", response_model=schemas.ItemResponse)
+def update_item(item_id: int, item_update: schemas.ItemCreate, db: Session = Depends(database.get_db)):
+    return ItemService.update_item(db, item_id, item_update)
+
+
+@router.delete("/{item_id}")
+def delete_item(item_id: int, db: Session = Depends(database.get_db)):
+    return ItemService.delete_item(db, item_id)
+
+
+@router.post("/{item_id}/check")
+@limiter.limit("10/minute")
+def check_item(
+    request: Request, item_id: int, background_tasks: BackgroundTasks, db: Session = Depends(database.get_db)
+):
+    item = ItemService.get_item(db, item_id)
+    if not item:
+        raise HTTPException(status_code=404, detail="Item not found")
+
+    item.is_refreshing = True
+    item.last_error = None
+    db.commit()
+
+    background_tasks.add_task(process_item_check, item_id)
+    return {"message": "Check triggered"}

app/routers/jobs.py

@@ -0,0 +1,61 @@
+from apscheduler.triggers.interval import IntervalTrigger
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
+from sqlalchemy.orm import Session
+
+from app import database, models, schemas
+from app.limiter import limiter
+from app.services.scheduler_service import process_item_check, scheduled_refresh, scheduler
+from app.services.settings_service import SettingsService
+
+router = APIRouter(prefix="/jobs", tags=["jobs"])
+
+
+@router.get("/config")
+def get_job_config(db: Session = Depends(database.get_db)):
+    interval = int(SettingsService.get_setting_value(db, "refresh_interval_minutes", "60"))
+
+    next_run = None
+    job = scheduler.get_job("refresh_job")
+    if job:
+        next_run = job.next_run_time
+
+    return {"refresh_interval_minutes": interval, "next_run": next_run, "running": scheduler.running}
+
+
+@router.post("/config")
+def update_job_config(config: schemas.SettingsUpdate, db: Session = Depends(database.get_db)):
+    if config.key != "refresh_interval_minutes":
+        raise HTTPException(status_code=400, detail="Invalid setting key for job config")
+
+    try:
+        interval = int(config.value)
+        if interval < 1:
+            raise ValueError("Interval must be at least 1 minute")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail="Invalid interval value") from e
+
+    SettingsService.update_setting(db, config)
+
+    try:
+        scheduler.reschedule_job("refresh_job", trigger=IntervalTrigger(minutes=interval))
+    except Exception:
+        scheduler.add_job(scheduled_refresh, IntervalTrigger(minutes=interval), id="refresh_job", replace_existing=True)
+
+    return {"message": "Job configuration updated", "refresh_interval_minutes": interval}
+
+
+@router.post("/refresh-all")
+@limiter.limit("5/minute")
+def refresh_all_items(request: Request, background_tasks: BackgroundTasks, db: Session = Depends(database.get_db)):
+    items = db.query(models.Item).filter(models.Item.is_active).all()
+
+    # Mark all items as refreshing immediately so UI updates persist
+    for item in items:
+        item.is_refreshing = True
+    db.commit()
+
+    for item in items:
+        if item.id is not None:
+            background_tasks.add_task(process_item_check, int(item.id))
+
+    return {"message": f"Triggered refresh for {len(items)} items"}

app/routers/notifications.py

@@ -0,0 +1,29 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+
+from app import database, schemas
+from app.services.notification_service import NotificationService
+
+router = APIRouter(prefix="/notification-profiles", tags=["notifications"])
+
+
+@router.get("", response_model=list[schemas.NotificationProfileResponse])
+def get_notification_profiles(db: Session = Depends(database.get_db)):
+    return NotificationService.get_notification_profiles(db)
+
+
+@router.post("", response_model=schemas.NotificationProfileResponse)
+def create_notification_profile(profile: schemas.NotificationProfileCreate, db: Session = Depends(database.get_db)):
+    return NotificationService.create_notification_profile(db, profile)
+
+
+@router.delete("/{profile_id}")
+def delete_notification_profile(profile_id: int, db: Session = Depends(database.get_db)):
+    return NotificationService.delete_notification_profile(db, profile_id)
+
+
+@router.put("/{profile_id}", response_model=schemas.NotificationProfileResponse)
+def update_notification_profile(
+    profile_id: int, profile: schemas.NotificationProfileUpdate, db: Session = Depends(database.get_db)
+):
+    return NotificationService.update_notification_profile(db, profile_id, profile)

app/routers/settings.py

@@ -0,0 +1,17 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+
+from app import database, schemas
+from app.services.settings_service import SettingsService
+
+router = APIRouter(prefix="/settings", tags=["settings"])
+
+
+@router.get("")
+def get_settings(db: Session = Depends(database.get_db)):
+    return SettingsService.get_settings(db)
+
+
+@router.post("", response_model=schemas.SettingsUpdate)
+def update_setting(setting: schemas.SettingsUpdate, db: Session = Depends(database.get_db)):
+    return SettingsService.update_setting(db, setting)

app/schemas.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict
+
+
+class NotificationProfileCreate(BaseModel):
+    name: str
+    apprise_url: str
+    notify_on_price_drop: bool = True
+    notify_on_target_price: bool = True
+    price_drop_threshold_percent: float = 10.0
+    notify_on_stock_change: bool = True
+    check_interval_minutes: int = 60
+
+
+class NotificationProfileUpdate(NotificationProfileCreate):
+    pass
+
+
+class NotificationProfileResponse(NotificationProfileCreate):
+    id: int
+    model_config = ConfigDict(from_attributes=True)
+
+
+class ItemCreate(BaseModel):
+    url: str
+    name: str
+    selector: str | None = None
+    target_price: float | None = None
+    check_interval_minutes: int = 60
+    tags: str | None = None
+    description: str | None = None
+    notification_profile_id: int | None = None
+
+
+class ItemResponse(ItemCreate):
+    id: int
+    current_price: float | None
+    in_stock: bool | None
+    current_price_confidence: float | None = None
+    in_stock_confidence: float | None = None
+    is_active: bool
+    last_checked: datetime | None
+    is_refreshing: bool = False
+    last_error: str | None = None
+    screenshot_url: str | None = None
+    model_config = ConfigDict(from_attributes=True)
+
+
+class SettingsUpdate(BaseModel):
+    key: str
+    value: str

app/services/ai_service.py

@@ -0,0 +1,352 @@
+import asyncio
+import json
+import logging
+import re
+import time
+from typing import Any, TypedDict
+
+from litellm import acompletion
+from pydantic import ValidationError
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from app import models
+from app.ai_schema import (
+    PROMPT_VERSION,
+    AIExtractionMetadata,
+    AIExtractionResponse,
+    get_extraction_prompt,
+    get_repair_prompt,
+)
+from app.database import SessionLocal
+from app.utils.image import encode_image
+from app.utils.text import clean_text
+
+# Default configuration (can be overridden by DB settings)
+DEFAULT_PROVIDER = "ollama"
+DEFAULT_MODEL = "gemma3:4b"
+DEFAULT_API_BASE = "http://ollama:11434"
+DEFAULT_TEMPERATURE = 0.1
+DEFAULT_MAX_TOKENS = 300
+DEFAULT_TIMEOUT = 30  # seconds
+MAX_TEXT_LENGTH = 5000  # Will be filtered to ~1500-2000 relevant chars
+
+# Config caching
+_config_cache: dict[str, Any] = {"data": None, "timestamp": 0.0}
+CONFIG_CACHE_TTL = 60  # seconds
+
+logger = logging.getLogger(__name__)
+
+
+class AIConfig(TypedDict):
+    provider: str
+    model: str
+    api_key: str
+    api_base: str
+    temperature: float
+    max_tokens: int
+    timeout: int
+    enable_json_repair: bool
+    enable_multi_sample: bool
+    multi_sample_threshold: float
+
+
+MIN_API_KEY_LENGTH = 12
+
+
+def _sanitize_api_key(key: str) -> str:
+    """Redact API key for logging (show first/last 4 chars only)."""
+    if not key or len(key) < MIN_API_KEY_LENGTH:
+        return "***"
+    return f"{key[:4]}...{key[-4:]}"
+
+
+class AIService:
+    @staticmethod
+    def get_ai_config() -> AIConfig:
+        """
+        Fetches AI configuration from the database with caching.
+        Returns a dict with: provider, model, api_key, api_base, temperature, max_tokens, timeout
+        """
+        # Check cache
+        now = time.time()
+        if _config_cache["data"] and (now - _config_cache["timestamp"]) < CONFIG_CACHE_TTL:
+            return _config_cache["data"]  # type: ignore
+
+        # Fetch from DB
+        session = SessionLocal()
+        try:
+            settings = session.query(models.Settings).all()
+            settings_map = {s.key: s.value for s in settings}
+
+            config: AIConfig = {
+                "provider": settings_map.get("ai_provider", DEFAULT_PROVIDER),
+                "model": settings_map.get("ai_model", DEFAULT_MODEL),
+                "api_key": settings_map.get("ai_api_key", ""),
+                "api_base": settings_map.get("ai_api_base", DEFAULT_API_BASE),
+                "temperature": float(settings_map.get("ai_temperature", str(DEFAULT_TEMPERATURE))),
+                "max_tokens": int(settings_map.get("ai_max_tokens", str(DEFAULT_MAX_TOKENS))),
+                "timeout": int(settings_map.get("ai_timeout", str(DEFAULT_TIMEOUT))),
+                "enable_json_repair": settings_map.get("enable_json_repair", "true").lower() == "true",
+                "enable_multi_sample": settings_map.get("enable_multi_sample", "false").lower() == "true",
+                "multi_sample_threshold": float(settings_map.get("multi_sample_confidence_threshold", "0.6")),
+            }
+
+            # Update cache
+            _config_cache["data"] = config  # type: ignore
+            _config_cache["timestamp"] = now
+
+            return config
+        except Exception as e:
+            logger.error(f"Error fetching AI config: {e}")
+            return {
+                "provider": DEFAULT_PROVIDER,
+                "model": DEFAULT_MODEL,
+                "api_key": "",
+                "api_base": DEFAULT_API_BASE,
+                "temperature": DEFAULT_TEMPERATURE,
+                "max_tokens": DEFAULT_MAX_TOKENS,
+                "timeout": DEFAULT_TIMEOUT,
+                "enable_json_repair": True,
+                "enable_multi_sample": False,
+                "multi_sample_threshold": 0.6,
+            }
+        finally:
+            session.close()
+
+    @staticmethod
+    def parse_and_validate_response(response_text: str) -> AIExtractionResponse:
+        """
+        Parse and validate AI response against schema.
+
+        Pipeline:
+        1. Extract JSON from response (handle markdown code blocks)
+        2. Parse JSON
+        3. Validate against Pydantic schema (includes normalization and clamping)
+
+        Raises:
+            ValidationError: If response doesn't match schema
+            json.JSONDecodeError: If JSON is invalid
+        """
+        # Try to extract JSON from markdown code blocks
+        json_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", response_text, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(1)
+        else:
+            # Try to find raw JSON object
+            json_match = re.search(r"\{.*\}", response_text, re.DOTALL)
+            if json_match:
+                json_str = json_match.group(0)
+            else:
+                json_str = response_text
+
+        # Parse JSON
+        data = json.loads(json_str)
+
+        # Validate and normalize through Pydantic
+        return AIExtractionResponse(**data)
+
+    @classmethod
+    async def repair_json_response(
+        cls,
+        raw_output: str,
+        config: AIConfig,
+    ) -> AIExtractionResponse:
+        """
+        Attempt to repair invalid JSON using a second LLM call.
+
+        Args:
+            raw_output: The raw AI output that failed parsing
+            config: AI configuration dict
+
+        Returns:
+            Validated AIExtractionResponse
+
+        Raises:
+            Exception: If repair also fails
+        """
+        logger.warning("Attempting JSON repair with second LLM call")
+
+        repair_prompt = get_repair_prompt(raw_output)
+
+        # Use a simpler, cheaper model for repair if possible
+        # For now, use the same model
+        kwargs = {
+            "model": config["model"] if config["provider"] != "ollama" else f"ollama/{config['model']}",
+            "messages": [{"role": "user", "content": repair_prompt}],
+            "max_tokens": 300,
+            "temperature": 0.0,  # Very deterministic for repair
+            "timeout": config["timeout"],
+        }
+
+        if config["api_key"]:
+            kwargs["api_key"] = config["api_key"]
+
+        if config["provider"] == "ollama":
+            kwargs["api_base"] = config["api_base"]
+            kwargs["format"] = "json"
+        elif config["provider"] == "openai" and config["api_base"]:
+            kwargs["api_base"] = config["api_base"]
+
+        response = await acompletion(**kwargs)
+        repaired_text = response.choices[0].message.content
+
+        # Validate repaired response
+        return cls.parse_and_validate_response(repaired_text)
+
+    @staticmethod
+    @retry(
+        retry=retry_if_exception_type((TimeoutError, ConnectionError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def call_llm(
+        prompt: str,
+        image_data_url: str,
+        config: AIConfig,
+    ) -> str:
+        """
+        Call LLM with structured output settings and retry logic.
+
+        Returns:
+            Raw response text from the model
+
+        Raises:
+            TimeoutError: If request times out after retries
+            ConnectionError: If connection fails after retries
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {"type": "image_url", "image_url": {"url": image_data_url}},
+                ],
+            }
+        ]
+
+        # Prepare kwargs for litellm
+        kwargs: dict[str, Any] = {
+            "model": config["model"] if config["provider"] != "ollama" else f"ollama/{config['model']}",
+            "messages": messages,
+            "max_tokens": config["max_tokens"],
+            "temperature": config["temperature"],
+            "timeout": config["timeout"],
+        }
+
+        if config["api_key"]:
+            kwargs["api_key"] = config["api_key"]
+
+        # Provider-specific structured output features
+        if config["provider"] == "ollama":
+            kwargs["api_base"] = config["api_base"]
+            kwargs["format"] = "json"  # Force JSON mode for Ollama
+        elif config["provider"] == "openai":
+            # Use OpenAI's JSON mode
+            kwargs["response_format"] = {"type": "json_object"}
+            if config["api_base"]:
+                kwargs["api_base"] = config["api_base"]
+        elif config["provider"] == "anthropic":
+            # Anthropic doesn't have native JSON mode yet, rely on prompt
+            if config["api_base"]:
+                kwargs["api_base"] = config["api_base"]
+        # Other providers - rely on prompt engineering
+        elif config["api_base"]:
+            kwargs["api_base"] = config["api_base"]
+
+        # Call litellm asynchronously
+        sanitized_key = _sanitize_api_key(config["api_key"]) if config["api_key"] else "(none)"
+        logger.info(
+            f"Calling {config['provider']}/{config['model']} "
+            f"(temp={config['temperature']}, max_tokens={config['max_tokens']}, "
+            f"timeout={config['timeout']}s, key={sanitized_key})"
+        )
+        response = await acompletion(**kwargs)
+
+        content = response.choices[0].message.content
+        return content or ""
+
+    @classmethod
+    async def analyze_image(
+        cls,
+        image_path: str,
+        page_text: str = "",
+    ) -> tuple[AIExtractionResponse, AIExtractionMetadata] | None:
+        """
+        Analyze image and extract price/stock information.
+
+        Args:
+            image_path: Path to screenshot
+            page_text: Optional webpage text context
+
+        Returns:
+            Tuple of (AIExtractionResponse, AIExtractionMetadata) or None on failure
+        """
+        try:
+            # Get AI config
+            loop = asyncio.get_running_loop()
+            config = await loop.run_in_executor(None, cls.get_ai_config)
+
+            logger.info(
+                f"Analyzing image with Provider: {config['provider']}, "
+                f"Model: {config['model']}, Timeout: {config['timeout']}s"
+            )
+
+            # Encode image
+            base64_image = await encode_image(image_path)
+            data_url = f"data:image/jpeg;base64,{base64_image}"
+
+            # Prepare prompt with optional text context
+            cleaned_text = ""
+            if page_text:
+                cleaned_text = clean_text(page_text)
+                if len(cleaned_text) > MAX_TEXT_LENGTH:
+                    cleaned_text = cleaned_text[:MAX_TEXT_LENGTH] + "...(truncated)"
+                logger.info(f"Added text context (original: {len(page_text)}, cleaned: {len(cleaned_text)})")
+
+            prompt = get_extraction_prompt(cleaned_text if cleaned_text else None)
+
+            # Call LLM
+            response_text = await cls.call_llm(prompt, data_url, config)
+
+            logger.info(f"AI Response: {response_text[:200]}...")
+
+            # Parse and validate response
+            repair_used = False
+            try:
+                extraction_result = cls.parse_and_validate_response(response_text)
+            except (ValidationError, json.JSONDecodeError) as e:
+                logger.warning(f"Primary parsing failed: {e}")
+
+                if config["enable_json_repair"]:
+                    try:
+                        extraction_result = await cls.repair_json_response(response_text, config)
+                        repair_used = True
+                        logger.info("JSON repair successful")
+                    except Exception as repair_error:
+                        logger.error(f"JSON repair also failed: {repair_error}")
+                        raise
+                else:
+                    raise
+
+            # Create metadata
+            metadata = AIExtractionMetadata(
+                model_name=config["model"],
+                provider=config["provider"],
+                prompt_version=PROMPT_VERSION,
+                repair_used=repair_used,
+                multi_sample=False,
+                sample_count=1,
+            )
+
+            logger.info(
+                f"Extraction successful: price={extraction_result.price} "
+                f"(conf={extraction_result.price_confidence:.2f}), "
+                f"stock={extraction_result.in_stock} (conf={extraction_result.in_stock_confidence:.2f})"
+            )
+
+            return extraction_result, metadata
+
+        except Exception as e:
+            logger.error(f"Error in analyze_image: {e}", exc_info=True)
+            return None

app/services/item_service.py

@@ -0,0 +1,135 @@
+import logging
+import os
+from datetime import UTC, datetime
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from app import models, schemas
+from app.services.settings_service import SettingsService
+from app.url_validation import URLValidationError, validate_url
+
+logger = logging.getLogger(__name__)
+
+
+class ItemService:
+    @staticmethod
+    def get_items(db: Session):
+        items = db.query(models.Item).all()
+        return [
+            {
+                **item.__dict__,
+                "screenshot_url": f"/screenshots/item_{item.id}.png"
+                if os.path.exists(f"screenshots/item_{item.id}.png")
+                else None,
+            }
+            for item in items
+        ]
+
+    @staticmethod
+    def create_item(db: Session, item: schemas.ItemCreate):
+        logger.info(f"Creating item: {item.name} - {item.url}")
+        try:
+            validate_url(item.url)
+        except URLValidationError as e:
+            raise HTTPException(status_code=400, detail=f"Invalid URL: {e}") from e
+
+        db_item = models.Item(**item.model_dump())
+        db.add(db_item)
+        db.commit()
+        db.refresh(db_item)
+        return db_item
+
+    @staticmethod
+    def update_item(db: Session, item_id: int, item_update: schemas.ItemCreate):
+        db_item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not db_item:
+            raise HTTPException(status_code=404, detail="Item not found")
+
+        for key, value in item_update.model_dump().items():
+            setattr(db_item, key, value)
+
+        db.commit()
+        db.refresh(db_item)
+        return db_item
+
+    @staticmethod
+    def delete_item(db: Session, item_id: int):
+        item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            raise HTTPException(status_code=404, detail="Item not found")
+
+        if os.path.exists(f"screenshots/item_{item_id}.png"):
+            try:
+                os.remove(f"screenshots/item_{item_id}.png")
+            except OSError:
+                pass
+
+        db.delete(item)
+        db.commit()
+        return {"ok": True}
+
+    @staticmethod
+    def get_item(db: Session, item_id: int):
+        return db.query(models.Item).filter(models.Item.id == item_id).first()
+
+    @staticmethod
+    def get_item_data_for_checking(db: Session, item_id: int):
+        item = db.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            return None, None
+
+        settings = {s.key: s.value for s in db.query(models.Settings).all()}
+        profile = item.notification_profile
+
+        item_data = {
+            "id": item.id,
+            "url": item.url,
+            "selector": item.selector,
+            "name": item.name,
+            "current_price": item.current_price,
+            "in_stock": item.in_stock,
+            "target_price": item.target_price,
+            "notification_profile": profile.__dict__ if profile else None,
+        }
+
+        config = {
+            "smart_scroll": settings.get("smart_scroll_enabled", "false").lower() == "true",
+            "smart_scroll_pixels": int(settings.get("smart_scroll_pixels", "350")),
+            "text_context_enabled": settings.get("text_context_enabled", "false").lower() == "true",
+            "text_length": int(settings.get("text_context_length", "5000"))
+            if settings.get("text_context_enabled", "false").lower() == "true"
+            else 0,
+            "scraper_timeout": int(settings.get("scraper_timeout", "90000")),
+        }
+        return item_data, config
+
+    @staticmethod
+    def get_due_items(db: Session):
+        items = db.query(models.Item).filter(models.Item.is_active).all()
+        global_interval = int(SettingsService.get_setting_value(db, "refresh_interval_minutes", "60"))
+        due_items = []
+        now = datetime.now(UTC)
+
+        for item in items:
+            if item.is_refreshing:
+                continue
+
+            interval = (
+                item.notification_profile.check_interval_minutes
+                if item.notification_profile and item.notification_profile.check_interval_minutes
+                else global_interval
+            )
+
+            if not item.last_checked:
+                due_items.append((item.id, interval, -1))
+                continue
+
+            last_checked = (
+                item.last_checked.replace(tzinfo=UTC) if item.last_checked.tzinfo is None else item.last_checked
+            )
+            time_since = (now - last_checked).total_seconds() / 60
+            if time_since >= interval:
+                due_items.append((item.id, interval, int(time_since)))
+
+        return due_items

app/services/notification_service.py

@@ -0,0 +1,89 @@
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from app import models, notification_sender, schemas
+
+
+class NotificationService:
+    @staticmethod
+    def get_notification_profiles(db: Session):
+        return db.query(models.NotificationProfile).all()
+
+    @staticmethod
+    def create_notification_profile(db: Session, profile: schemas.NotificationProfileCreate):
+        db_profile = models.NotificationProfile(**profile.model_dump())
+        db.add(db_profile)
+        db.commit()
+        db.refresh(db_profile)
+        return db_profile
+
+    @staticmethod
+    def delete_notification_profile(db: Session, profile_id: int):
+        profile = db.query(models.NotificationProfile).filter(models.NotificationProfile.id == profile_id).first()
+        if not profile:
+            raise HTTPException(status_code=404, detail="Profile not found")
+        db.delete(profile)
+        db.commit()
+        db.delete(profile)
+        db.commit()
+        return {"ok": True}
+
+    @staticmethod
+    def update_notification_profile(db: Session, profile_id: int, profile_data: schemas.NotificationProfileUpdate):
+        profile = db.query(models.NotificationProfile).filter(models.NotificationProfile.id == profile_id).first()
+        if not profile:
+            raise HTTPException(status_code=404, detail="Profile not found")
+
+        for key, value in profile_data.model_dump().items():
+            setattr(profile, key, value)
+
+        db.commit()
+        db.refresh(profile)
+        return profile
+
+    @staticmethod
+    async def send_item_notifications(
+        item_data: dict,
+        price: float | None,
+        old_price: float | None,
+        in_stock: bool | None,
+        old_stock: bool | None,
+    ):
+        profile = item_data["notification_profile"]
+        if not profile:
+            return
+
+        if profile["notify_on_price_drop"] and price is not None and old_price is not None:
+            if price < old_price:
+                drop_percent = ((old_price - price) / old_price) * 100
+                if drop_percent >= profile["price_drop_threshold_percent"]:
+                    await notification_sender.send_notification(
+                        [profile["apprise_url"]],
+                        f"Price Drop Alert: {item_data['name']}",
+                        f"Price dropped by {drop_percent:.1f}%! Now ${price} (was ${old_price})",
+                    )
+
+        if (
+            profile["notify_on_target_price"]
+            and price is not None
+            and item_data["target_price"]
+            and price <= item_data["target_price"]
+        ):
+            await notification_sender.send_notification(
+                [profile["apprise_url"]],
+                f"Target Price Alert: {item_data['name']}",
+                f"Price is ${price} (Target: ${item_data['target_price']})",
+            )
+
+        if (
+            profile["notify_on_stock_change"]
+            and in_stock is not None
+            and old_stock is not None
+            and in_stock != old_stock
+        ):
+            status = "In Stock" if in_stock else "Out of Stock"
+            await notification_sender.send_notification(
+                [profile["apprise_url"]],
+                f"Stock Alert: {item_data['name']}",
+                f"Item is now {status}",
+            )

app/services/scheduler_service.py

@@ -0,0 +1,147 @@
+import asyncio
+import logging
+from datetime import UTC, datetime
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from app import database, models
+from app.ai_schema import AIExtractionMetadata, AIExtractionResponse
+from app.services.ai_service import AIService
+from app.services.item_service import ItemService
+from app.services.notification_service import NotificationService
+from app.services.scraper_service import ScraperService
+
+logger = logging.getLogger(__name__)
+scheduler = AsyncIOScheduler()
+
+PRICE_CHANGE_THRESHOLD_PERCENT = 20.0
+LOW_CONFIDENCE_THRESHOLD = 0.7
+
+
+def _get_thresholds():
+    with database.SessionLocal() as session:
+        settings = {s.key: s.value for s in session.query(models.Settings).all()}
+        return {
+            "price": float(settings.get("confidence_threshold_price", "0.5")),
+            "stock": float(settings.get("confidence_threshold_stock", "0.5")),
+        }
+
+
+def _update_db_result(
+    item_id: int,
+    extraction: AIExtractionResponse,
+    metadata: AIExtractionMetadata,
+    thresholds: dict,
+    screenshot_path: str,
+):
+    with database.SessionLocal() as session:
+        item = session.query(models.Item).filter(models.Item.id == item_id).first()
+        if not item:
+            return None, None
+
+        old_price, old_stock = item.current_price, item.in_stock
+        price, in_stock = extraction.price, extraction.in_stock
+        p_conf, s_conf = extraction.price_confidence, extraction.in_stock_confidence
+
+        if price is not None:
+            if p_conf >= thresholds["price"]:
+                if (
+                    old_price
+                    and (abs(price - old_price) / old_price * 100 > PRICE_CHANGE_THRESHOLD_PERCENT)
+                    and p_conf < LOW_CONFIDENCE_THRESHOLD
+                ):
+                    item.last_error = f"Uncertain: Large price change with low confidence ({p_conf:.2f})"
+                else:
+                    item.last_error = None
+                item.current_price = price
+                item.current_price_confidence = p_conf
+
+            session.add(
+                models.PriceHistory(
+                    item_id=item.id,
+                    price=price,
+                    screenshot_path=screenshot_path,
+                    price_confidence=p_conf,
+                    in_stock_confidence=s_conf,
+                    ai_model=metadata.model_name,
+                    ai_provider=metadata.provider,
+                    prompt_version=metadata.prompt_version,
+                    repair_used=metadata.repair_used,
+                )
+            )
+
+        if in_stock is not None and s_conf >= thresholds["stock"]:
+            item.in_stock = in_stock
+            item.in_stock_confidence = s_conf
+
+        item.last_checked = datetime.now(UTC)
+        item.is_refreshing = False
+        if item.last_error and not item.last_error.startswith("Uncertain:"):
+            item.last_error = None
+
+        session.commit()
+        return old_price, old_stock
+
+
+def _update_db_error(item_id, error_msg):
+    with database.SessionLocal() as session:
+        if item := session.query(models.Item).filter(models.Item.id == item_id).first():
+            item.is_refreshing = False
+            item.last_error = error_msg
+            item.last_checked = datetime.now(UTC)
+            session.commit()
+
+
+async def process_item_check(item_id: int):
+    loop = asyncio.get_running_loop()
+    with database.SessionLocal() as session:
+        item_data, config = await loop.run_in_executor(None, ItemService.get_item_data_for_checking, session, item_id)
+
+    if not item_data:
+        logger.error(f"process_item_check: Item ID {item_id} not found")
+        return
+
+    try:
+        logger.info(f"Checking item: {item_data['name']} ({item_data['url']})")
+        screenshot_path, page_text = await ScraperService.scrape_item(
+            item_data["url"],
+            item_data["selector"],
+            item_id,
+            smart_scroll=config["smart_scroll"],
+            scroll_pixels=config["smart_scroll_pixels"],
+            text_length=config["text_length"],
+            timeout=config["scraper_timeout"],
+        )
+
+        if not screenshot_path:
+            raise Exception("Failed to capture screenshot")
+
+        if not (ai_result := await AIService.analyze_image(screenshot_path, page_text=page_text)):
+            raise Exception("AI analysis failed")
+
+        extraction, metadata = ai_result
+        thresholds = await loop.run_in_executor(None, _get_thresholds)
+        old_price, old_stock = await loop.run_in_executor(
+            None, _update_db_result, item_id, extraction, metadata, thresholds, screenshot_path
+        )
+
+        await NotificationService.send_item_notifications(
+            item_data, extraction.price, old_price, extraction.in_stock, old_stock
+        )
+
+    except Exception as e:
+        logger.error(f"Error in process_item_check: {e}")
+        await loop.run_in_executor(None, _update_db_error, item_id, str(e))
+
+
+async def scheduled_refresh():
+    logger.info("Heartbeat: Checking for items due for refresh")
+    loop = asyncio.get_running_loop()
+    try:
+        with database.SessionLocal() as session:
+            due_items = await loop.run_in_executor(None, ItemService.get_due_items, session)
+
+        for item_id, _, _ in due_items:
+            await process_item_check(item_id)
+    except Exception as e:
+        logger.error(f"Error in scheduled refresh: {e}", exc_info=True)

app/services/scraper_service.py

@@ -0,0 +1,185 @@
+import asyncio
+import logging
+import os
+
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright.async_api import async_playwright
+
+logger = logging.getLogger(__name__)
+
+BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
+
+
+class ScraperService:
+    @staticmethod
+    async def scrape_item(  # noqa: PLR0913, PLR0912, PLR0915
+        url: str,
+        selector: str | None = None,
+        item_id: int | None = None,
+        smart_scroll: bool = False,
+        scroll_pixels: int = 350,
+        text_length: int = 0,
+        timeout: int = 90000,
+    ) -> tuple[str | None, str]:
+        """
+        Scrapes the given URL using Browserless and Playwright.
+        Returns a tuple: (screenshot_path, page_text)
+
+        Args:
+            url: Target URL to scrape
+            selector: Optional CSS selector to focus on
+            item_id: Optional item ID for screenshot naming
+            smart_scroll: Enable scrolling to load lazy content
+            scroll_pixels: Number of pixels to scroll (must be positive)
+            text_length: Number of characters to extract (0 = disabled)
+            timeout: Page load timeout in milliseconds
+        """
+        # Input validation
+        if scroll_pixels <= 0:
+            logger.warning(f"Invalid scroll_pixels value: {scroll_pixels}, using default 350")
+            scroll_pixels = 350
+
+        if timeout <= 0:
+            logger.warning(f"Invalid timeout value: {timeout}, using default 90000")
+            timeout = 90000
+        async with async_playwright() as p:
+            browser = None
+            try:
+                logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}")
+                browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL)
+                context = await browser.new_context(
+                    viewport={"width": 1920, "height": 1080},
+                    user_agent=(
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                        "AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/120.0.0.0 Safari/537.36"
+                    ),
+                )
+
+                # Stealth mode / Ad blocking attempts
+                await context.route("**/*", lambda route: route.continue_())
+
+                page = await context.new_page()
+
+                logger.info(f"Navigating to {url} (Timeout: {timeout}ms)")
+                try:
+                    # First wait for domcontentloaded - this is the minimum we need
+                    await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
+                    logger.info(f"Page loaded (domcontentloaded): {url}")
+
+                    # Then try to wait for networkidle, but don't fail if it times out
+                    # This helps with heavy pages that never fully settle
+                    try:
+                        await page.wait_for_load_state("networkidle", timeout=5000)
+                        logger.info("Network idle reached")
+                    except PlaywrightTimeoutError:
+                        logger.info("Network idle timed out (non-critical), proceeding...")
+
+                except Exception as e:
+                    logger.error(f"Error navigating to {url}: {e}")
+                    # Try to take screenshot anyway if page partially loaded
+                    pass
+
+                # Wait a bit for dynamic content if needed
+                await page.wait_for_timeout(2000)
+
+                # Try to close common popups
+                logger.info("Attempting to close popups...")
+                popup_selectors = [
+                    "button[aria-label='Close']",
+                    "button[aria-label='close']",
+                    ".close-button",
+                    ".modal-close",
+                    "svg[data-name='Close']",
+                    "[class*='popup'] button",
+                    "[class*='modal'] button",
+                    "button:has-text('No, thanks')",
+                    "button:has-text('No thanks')",
+                    "a:has-text('No, thanks')",
+                    "div[role='dialog'] button[aria-label='Close']",
+                ]
+
+                for popup_selector in popup_selectors:
+                    try:
+                        if await page.locator(popup_selector).count() > 0:
+                            logger.info(f"Found popup close button: {popup_selector}")
+                            # Try to click it. If it fails, catch and continue
+                            await page.locator(popup_selector).first.click(timeout=2000)
+                            await page.wait_for_timeout(1000)  # Wait for animation
+                    except Exception as e:
+                        logger.debug(f"Could not close popup with selector {popup_selector}: {e}")
+
+                # Also try pressing Escape
+                try:
+                    await page.keyboard.press("Escape")
+                except Exception as e:
+                    logger.debug(f"Could not press Escape key: {e}")
+
+                if selector:
+                    try:
+                        logger.info(f"Waiting for selector: {selector}")
+                        await page.wait_for_selector(selector, timeout=5000)
+                        # Scroll to element
+                        element = page.locator(selector).first
+                        await element.scroll_into_view_if_needed()
+                        logger.info(f"Scrolled to selector: {selector}")
+                    except Exception as e:
+                        logger.warning(f"Selector {selector} not found or timed out: {e}")
+                else:
+                    # Auto-detect price if no selector
+                    logger.info("No selector provided. Attempting to find price element...")
+                    try:
+                        # Look for common price patterns
+                        price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/")
+                        if await price_locator.count() > 0:
+                            # Pick the first one that looks visible and reasonable size
+                            # This is heuristic
+                            await price_locator.first.scroll_into_view_if_needed()
+                            logger.info("Scrolled to potential price element")
+                    except Exception as e:
+                        logger.warning(f"Auto-price detection failed: {e}")
+
+                # Smart Scroll
+                if smart_scroll:
+                    logger.info(f"Performing smart scroll ({scroll_pixels}px)...")
+                    try:
+                        await page.evaluate(f"window.scrollBy(0, {scroll_pixels})")
+                        await page.wait_for_timeout(1000)
+                    except Exception as e:
+                        logger.warning(f"Smart scroll failed: {e}")
+
+                # Text Extraction
+                page_text = ""
+                if text_length > 0:
+                    try:
+                        logger.info(f"Extracting text (limit: {text_length} chars)...")
+                        # Get text from body
+                        raw_text = await page.inner_text("body")
+                        # Simple truncation
+                        page_text = raw_text[:text_length]
+                        logger.info(f"Extracted {len(page_text)} characters")
+                    except Exception as e:
+                        logger.error(f"Text extraction failed: {e}")
+
+                # Take screenshot
+                screenshot_dir = "screenshots"
+                os.makedirs(screenshot_dir, exist_ok=True)
+                if item_id:
+                    filename = f"{screenshot_dir}/item_{item_id}.png"
+                else:
+                    url_part = url.split("//")[-1].replace("/", "_")
+                    timestamp = asyncio.get_event_loop().time()
+                    filename = f"{screenshot_dir}/{url_part}_{timestamp}.png"
+
+                await page.screenshot(path=filename, full_page=False)
+                logger.info(f"Screenshot saved to {filename}")
+
+                return filename, page_text
+
+            except Exception as e:
+                logger.error(f"Error scraping {url}: {e}")
+                return None, ""
+
+            finally:
+                if browser:
+                    await browser.close()

app/services/settings_service.py

@@ -0,0 +1,25 @@
+from sqlalchemy.orm import Session
+
+from app import models, schemas
+
+
+class SettingsService:
+    @staticmethod
+    def get_settings(db: Session):
+        return db.query(models.Settings).all()
+
+    @staticmethod
+    def update_setting(db: Session, setting: schemas.SettingsUpdate):
+        db_setting = db.query(models.Settings).filter(models.Settings.key == setting.key).first()
+        if db_setting:
+            db_setting.value = setting.value
+        else:
+            db_setting = models.Settings(key=setting.key, value=setting.value)
+            db.add(db_setting)
+        db.commit()
+        return db_setting
+
+    @staticmethod
+    def get_setting_value(db: Session, key: str, default: str | None = None):
+        setting = db.query(models.Settings).filter(models.Settings.key == key).first()
+        return setting.value if setting else default

app/url_validation.py

@@ -0,0 +1,69 @@
+"""URL validation utilities for security."""
+
+import re
+from urllib.parse import urlparse
+
+
+class URLValidationError(Exception):
+    """Raised when URL validation fails."""
+
+    pass
+
+
+# Blocked schemes that could be used for SSRF
+BLOCKED_SCHEMES = {"file", "javascript", "data", "ftp"}
+
+# Private IP ranges (RFC 1918, RFC 4193, loopback)
+PRIVATE_IP_PATTERNS = [
+    re.compile(r"^127\."),  # Loopback
+    re.compile(r"^10\."),  # Private Class A
+    re.compile(r"^172\.(1[6-9]|2[0-9]|3[0-1])\."),  # Private Class B
+    re.compile(r"^192\.168\."),  # Private Class C
+    re.compile(r"^169\.254\."),  # Link-local
+    re.compile(r"^::1$"),  # IPv6 loopback
+    re.compile(r"^fc00:"),  # IPv6 private
+    re.compile(r"^fe80:"),  # IPv6 link-local
+    re.compile(r"^localhost$", re.IGNORECASE),
+    re.compile(r"^0\.0\.0\.0$"),  # Any address
+]
+
+
+def validate_url(url: str, allow_private: bool = False) -> None:
+    """
+    Validate URL for security (SSRF prevention).
+
+    Args:
+        url: URL to validate
+        allow_private: Allow private/internal IPs (default: False)
+
+    Raises:
+        URLValidationError: If URL is invalid or blocked
+    """
+    if not url or not isinstance(url, str):
+        raise URLValidationError("URL must be a non-empty string")
+
+    try:
+        parsed = urlparse(url.strip())
+    except Exception as e:
+        raise URLValidationError(f"Invalid URL format: {e}") from e
+
+    # Check scheme
+    if not parsed.scheme:
+        raise URLValidationError("URL must include a scheme (http:// or https://)")
+
+    if parsed.scheme.lower() in BLOCKED_SCHEMES:
+        raise URLValidationError(f"Blocked URL scheme: {parsed.scheme}")
+
+    if parsed.scheme.lower() not in {"http", "https"}:
+        raise URLValidationError(f"Only HTTP/HTTPS schemes allowed, got: {parsed.scheme}")
+
+    # Check hostname
+    if not parsed.hostname:
+        raise URLValidationError("URL must include a hostname")
+
+    # Check for private IPs unless explicitly allowed
+    if not allow_private:
+        hostname = parsed.hostname.lower()
+        for pattern in PRIVATE_IP_PATTERNS:
+            if pattern.match(hostname):
+                raise URLValidationError(f"Private/internal IPs not allowed: {hostname}")

app/utils/image.py

@@ -0,0 +1,45 @@
+import asyncio
+import base64
+import io
+import logging
+
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+# Image processing constants
+MAX_IMAGE_SIZE = 1024
+JPEG_QUALITY = 85
+
+
+def _process_image(image_path: str) -> str:
+    """
+    Synchronous image processing function to be run in an executor.
+    """
+    try:
+        with Image.open(image_path) as img:
+            # Resize if too large (e.g., max dimension 1024)
+            if max(img.size) > MAX_IMAGE_SIZE:
+                img.thumbnail((MAX_IMAGE_SIZE, MAX_IMAGE_SIZE))
+                logger.info(f"Resized image to {img.size}")
+
+            # Convert to RGB if necessary (e.g. for PNGs with alpha)
+            if img.mode in ("RGBA", "P"):
+                img_to_process = img.convert("RGB")
+            else:
+                img_to_process = img
+
+            buffered = io.BytesIO()
+            img_to_process.save(buffered, format="JPEG", quality=JPEG_QUALITY)
+            return base64.b64encode(buffered.getvalue()).decode("utf-8")
+    except Exception as e:
+        logger.error(f"Error encoding image: {e}")
+        raise
+
+
+async def encode_image(image_path: str) -> str:
+    """
+    Asynchronously encode image by running blocking code in a thread.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, _process_image, image_path)

app/utils/text.py

@@ -0,0 +1,144 @@
+import re
+
+# Text filtering constants
+MIN_SNIPPET_LENGTH = 10
+SNIPPET_MERGE_DISTANCE = 50
+SNIPPET_CONTEXT_WINDOW = 100
+
+
+def clean_text(text: str) -> str:
+    """
+    Cleans the text by removing code blocks, HTML tags, and excessive whitespace.
+    """
+    if not text:
+        return ""
+
+    # Remove code blocks (```...```)
+    text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
+
+    # Remove HTML tags (basic)
+    text = re.sub(r"<[^>]+>", "", text)
+
+    # Remove non-printable characters (keep newlines and tabs)
+    text = re.sub(r"[^\x20-\x7E\n\t]", "", text)
+
+    # Collapse excessive whitespace
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text
+
+
+def _find_matches(text: str, text_lower: str, keyword: str) -> list[tuple[int, str]]:
+    snippets = []
+    if keyword.startswith("r\\"):
+        pattern = keyword[1:]  # Remove 'r' prefix
+        for match in re.finditer(pattern, text_lower, re.IGNORECASE):
+            start = max(0, match.start() - SNIPPET_CONTEXT_WINDOW)
+            end = min(len(text), match.end() + SNIPPET_CONTEXT_WINDOW)
+            snippet = text[start:end].strip()
+            if snippet and len(snippet) > MIN_SNIPPET_LENGTH:
+                snippets.append((start, snippet))
+    else:
+        pos = 0
+        while True:
+            pos = text_lower.find(keyword.lower(), pos)
+            if pos == -1:
+                break
+            start = max(0, pos - SNIPPET_CONTEXT_WINDOW)
+            end = min(len(text), pos + len(keyword) + SNIPPET_CONTEXT_WINDOW)
+            snippet = text[start:end].strip()
+            if snippet and len(snippet) > MIN_SNIPPET_LENGTH:
+                snippets.append((start, snippet))
+            pos += 1
+    return snippets
+
+
+def filter_relevant_text(text: str, max_length: int = 2000) -> str:
+    """
+    Filter text to extract only relevant snippets around price and stock indicators.
+
+    Args:
+        text: Full cleaned webpage text
+        max_length: Maximum total length of filtered output
+
+    Returns:
+        Filtered text containing only relevant snippets
+    """
+    if not text:
+        return ""
+
+    # Keywords to search for (case-insensitive)
+    price_keywords = [
+        r"\$\d+\.?\d*",  # $XX.XX pattern
+        r"\d+\.\d{2}\s*(usd|eur|gbp|cad)",  # XX.XX USD pattern
+        "price:",
+        "cost:",
+        "sale:",
+        "msrp:",
+        "save:",
+        "discount:",
+        r"\$",  # Any dollar sign
+    ]
+
+    stock_keywords = [
+        "add to cart",
+        "buy now",
+        "purchase",
+        "order now",
+        "in stock",
+        "out of stock",
+        "available",
+        "unavailable",
+        "sold out",
+        "notify me",
+        "back in stock",
+        "pre-order",
+        "ships",
+        "delivery",
+        "get it by",
+    ]
+
+    all_keywords = price_keywords + stock_keywords
+    snippets = []
+
+    text_lower = text.lower()
+
+    # Find all matches and extract context
+    for keyword in all_keywords:
+        snippets.extend(_find_matches(text, text_lower, keyword))
+
+    if not snippets:
+        # No matches found, return beginning of text
+        if len(text) > max_length:
+            return text[:max_length] + "...(truncated)"
+        return text
+
+    # Sort by position and deduplicate overlapping snippets
+    snippets.sort(key=lambda x: x[0])
+    merged_snippets = []
+    current_start, current_text = snippets[0]
+    current_end = current_start + len(current_text)
+
+    for start, snippet in snippets[1:]:
+        end = start + len(snippet)
+        # If overlapping or close together, merge
+        if start <= current_end + SNIPPET_MERGE_DISTANCE:
+            # Extend current snippet
+            if end > current_end:
+                # Merge overlapping text
+                current_text = current_text + " " + snippet[max(0, current_end - start) :]
+                current_end = end
+        else:
+            # Save current and start new
+            merged_snippets.append(current_text)
+            current_start, current_text = start, snippet
+            current_end = end
+
+    merged_snippets.append(current_text)
+
+    # Join snippets with separator and limit total length
+    result = " ... ".join(merged_snippets)
+    if len(result) > max_length:
+        result = result[:max_length] + "...(truncated)"
+
+    return result