mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-19 13:08:15 +02:00
perf: fix model reload thrash and cut time-to-first-token
Ollama reloaded the chat model mid-message because plan/summary/router calls sent divergent runner options (no num_ctx/num_batch) and omitted keep_alive — the main cause of perceived slowness. - share one keep-alive httpx.AsyncClient for all Ollama calls - unify runner options (runner_options) + keep_alive on every model call, embeddings included - drop the blocking LLM routing fallback (pure lexical heuristic) - run RAG recall + plan + code-model pick in parallel inside the SSE stream, after the start event - RAG cosine scoring off the event loop; cache /api/tags 30s and nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup config fetch feat: working session menu in TopBar (switch/create/rename/delete) feat: workspace file deletion (DELETE /api/files + UI trash buttons) docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
552548e4b4
commit
cb2872c78a
23 files changed
+672
-224
No files matched your search
+14
-3
@@ -52,8 +52,18 @@ def build_convo(sid: str, system_prompt: str) -> list[dict]:
|
||||
return convo
|
||||
|
||||
|
||||
async def maybe_summarize(sid: str, model: str) -> None:
|
||||
"""Compresse les anciens tours dans le résumé (tâche d'arrière-plan)."""
|
||||
async def maybe_summarize(
|
||||
sid: str,
|
||||
model: str,
|
||||
*,
|
||||
options: dict | None = None,
|
||||
keep_alive: str | None = None,
|
||||
) -> None:
|
||||
"""Compresse les anciens tours dans le résumé (tâche d'arrière-plan).
|
||||
|
||||
``options`` doit reprendre les options runner du chat (num_ctx…) pour ne
|
||||
pas déclencher un rechargement du modèle après chaque réponse.
|
||||
"""
|
||||
try:
|
||||
messages = db.list_messages_for_model(sid)
|
||||
if len(messages) <= SUMMARIZE_AFTER:
|
||||
@@ -76,7 +86,8 @@ async def maybe_summarize(sid: str, model: str) -> None:
|
||||
{"role": "system", "content": _SUMMARY_PROMPT},
|
||||
{"role": "user", "content": transcript},
|
||||
],
|
||||
options={"temperature": 0.2, "num_predict": 350},
|
||||
options={**(options or {}), "temperature": 0.2, "num_predict": 350},
|
||||
keep_alive=keep_alive,
|
||||
stream=True,
|
||||
):
|
||||
text += chunk.get("message", {}).get("content", "")
|
||||
|
||||
Reference in new issue
Block a user