mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-20 21:48:47 +02:00
perf: fix model reload thrash and cut time-to-first-token
Ollama reloaded the chat model mid-message because plan/summary/router calls sent divergent runner options (no num_ctx/num_batch) and omitted keep_alive — the main cause of perceived slowness. - share one keep-alive httpx.AsyncClient for all Ollama calls - unify runner options (runner_options) + keep_alive on every model call, embeddings included - drop the blocking LLM routing fallback (pure lexical heuristic) - run RAG recall + plan + code-model pick in parallel inside the SSE stream, after the start event - RAG cosine scoring off the event loop; cache /api/tags 30s and nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup config fetch feat: working session menu in TopBar (switch/create/rename/delete) feat: workspace file deletion (DELETE /api/files + UI trash buttons) docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
552548e4b4
commit
cb2872c78a
23 files changed
+672
-224
No files matched your search
+21
-11
@@ -9,6 +9,7 @@ silencieusement (aucun impact sur le chat).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
@@ -60,7 +61,7 @@ async def resolve_embed_model(preference: str | None = None) -> str | None:
|
||||
|
||||
value = None
|
||||
try:
|
||||
for m in await ollama.list_models():
|
||||
for m in await ollama.list_models_cached():
|
||||
name = (m.get("name") or "").lower()
|
||||
if any(h in name for h in _EMBED_HINTS):
|
||||
value = m["name"]
|
||||
@@ -79,6 +80,24 @@ def _cosine(a: list[float], b: list[float]) -> float:
|
||||
return dot / (na * nb) if na and nb else 0.0
|
||||
|
||||
|
||||
def _score_rows(qvec: list[float], rows: list) -> list[str]:
|
||||
"""Scoring cosinus sur toutes les mémoires — CPU pur, à lancer via to_thread.
|
||||
|
||||
Jusqu'à _MAX_MEMORIES vecteurs : la boucle Python bloquerait l'event loop
|
||||
(et donc tous les SSE en cours) pendant plusieurs dizaines de ms.
|
||||
"""
|
||||
scored: list[tuple[float, str]] = []
|
||||
for row in rows:
|
||||
try:
|
||||
score = _cosine(qvec, json.loads(row["embedding"]))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if score >= _MIN_SCORE:
|
||||
scored.append((score, row["content"]))
|
||||
scored.sort(reverse=True)
|
||||
return [c for _, c in scored[:_TOP_K]]
|
||||
|
||||
|
||||
async def index_exchange(
|
||||
sid: str, user_text: str, assistant_text: str, *, embed_model: str | None
|
||||
) -> None:
|
||||
@@ -127,16 +146,7 @@ async def recall(
|
||||
(sid,),
|
||||
).fetchall()
|
||||
|
||||
scored = []
|
||||
for row in rows:
|
||||
try:
|
||||
score = _cosine(qvec, json.loads(row["embedding"]))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if score >= _MIN_SCORE:
|
||||
scored.append((score, row["content"]))
|
||||
scored.sort(reverse=True)
|
||||
return [c for _, c in scored[:_TOP_K]]
|
||||
return await asyncio.to_thread(_score_rows, qvec, rows)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning("Rappel RAG impossible : %s", exc)
|
||||
return []
|
||||
Reference in new issue
Block a user