Add files via upload

app/utils/text.py

@@ -0,0 +1,144 @@
+import re
+
+# Text filtering constants
+MIN_SNIPPET_LENGTH = 10
+SNIPPET_MERGE_DISTANCE = 50
+SNIPPET_CONTEXT_WINDOW = 100
+
+
+def clean_text(text: str) -> str:
+    """
+    Cleans the text by removing code blocks, HTML tags, and excessive whitespace.
+    """
+    if not text:
+        return ""
+
+    # Remove code blocks (```...```)
+    text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
+
+    # Remove HTML tags (basic)
+    text = re.sub(r"<[^>]+>", "", text)
+
+    # Remove non-printable characters (keep newlines and tabs)
+    text = re.sub(r"[^\x20-\x7E\n\t]", "", text)
+
+    # Collapse excessive whitespace
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text
+
+
+def _find_matches(text: str, text_lower: str, keyword: str) -> list[tuple[int, str]]:
+    snippets = []
+    if keyword.startswith("r\\"):
+        pattern = keyword[1:]  # Remove 'r' prefix
+        for match in re.finditer(pattern, text_lower, re.IGNORECASE):
+            start = max(0, match.start() - SNIPPET_CONTEXT_WINDOW)
+            end = min(len(text), match.end() + SNIPPET_CONTEXT_WINDOW)
+            snippet = text[start:end].strip()
+            if snippet and len(snippet) > MIN_SNIPPET_LENGTH:
+                snippets.append((start, snippet))
+    else:
+        pos = 0
+        while True:
+            pos = text_lower.find(keyword.lower(), pos)
+            if pos == -1:
+                break
+            start = max(0, pos - SNIPPET_CONTEXT_WINDOW)
+            end = min(len(text), pos + len(keyword) + SNIPPET_CONTEXT_WINDOW)
+            snippet = text[start:end].strip()
+            if snippet and len(snippet) > MIN_SNIPPET_LENGTH:
+                snippets.append((start, snippet))
+            pos += 1
+    return snippets
+
+
+def filter_relevant_text(text: str, max_length: int = 2000) -> str:
+    """
+    Filter text to extract only relevant snippets around price and stock indicators.
+
+    Args:
+        text: Full cleaned webpage text
+        max_length: Maximum total length of filtered output
+
+    Returns:
+        Filtered text containing only relevant snippets
+    """
+    if not text:
+        return ""
+
+    # Keywords to search for (case-insensitive)
+    price_keywords = [
+        r"\$\d+\.?\d*",  # $XX.XX pattern
+        r"\d+\.\d{2}\s*(usd|eur|gbp|cad)",  # XX.XX USD pattern
+        "price:",
+        "cost:",
+        "sale:",
+        "msrp:",
+        "save:",
+        "discount:",
+        r"\$",  # Any dollar sign
+    ]
+
+    stock_keywords = [
+        "add to cart",
+        "buy now",
+        "purchase",
+        "order now",
+        "in stock",
+        "out of stock",
+        "available",
+        "unavailable",
+        "sold out",
+        "notify me",
+        "back in stock",
+        "pre-order",
+        "ships",
+        "delivery",
+        "get it by",
+    ]
+
+    all_keywords = price_keywords + stock_keywords
+    snippets = []
+
+    text_lower = text.lower()
+
+    # Find all matches and extract context
+    for keyword in all_keywords:
+        snippets.extend(_find_matches(text, text_lower, keyword))
+
+    if not snippets:
+        # No matches found, return beginning of text
+        if len(text) > max_length:
+            return text[:max_length] + "...(truncated)"
+        return text
+
+    # Sort by position and deduplicate overlapping snippets
+    snippets.sort(key=lambda x: x[0])
+    merged_snippets = []
+    current_start, current_text = snippets[0]
+    current_end = current_start + len(current_text)
+
+    for start, snippet in snippets[1:]:
+        end = start + len(snippet)
+        # If overlapping or close together, merge
+        if start <= current_end + SNIPPET_MERGE_DISTANCE:
+            # Extend current snippet
+            if end > current_end:
+                # Merge overlapping text
+                current_text = current_text + " " + snippet[max(0, current_end - start) :]
+                current_end = end
+        else:
+            # Save current and start new
+            merged_snippets.append(current_text)
+            current_start, current_text = start, snippet
+            current_end = end
+
+    merged_snippets.append(current_text)
+
+    # Join snippets with separator and limit total length
+    result = " ... ".join(merged_snippets)
+    if len(result) > max_length:
+        result = result[:max_length] + "...(truncated)"
+
+    return result