mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-22 06:28:11 +02:00
perf: fix model reload thrash and cut time-to-first-token
Ollama reloaded the chat model mid-message because plan/summary/router calls sent divergent runner options (no num_ctx/num_batch) and omitted keep_alive — the main cause of perceived slowness. - share one keep-alive httpx.AsyncClient for all Ollama calls - unify runner options (runner_options) + keep_alive on every model call, embeddings included - drop the blocking LLM routing fallback (pure lexical heuristic) - run RAG recall + plan + code-model pick in parallel inside the SSE stream, after the start event - RAG cosine scoring off the event loop; cache /api/tags 30s and nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup config fetch feat: working session menu in TopBar (switch/create/rename/delete) feat: workspace file deletion (DELETE /api/files + UI trash buttons) docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
552548e4b4
commit
cb2872c78a
23 files changed
+672
-224
No files matched your search
@@ -179,15 +179,15 @@ def save_config(patch: dict, model: str | None = None) -> dict:
|
||||
return get_config(model)
|
||||
|
||||
|
||||
def ollama_options(cfg: dict) -> dict:
|
||||
"""Traduit la config en options de génération Ollama."""
|
||||
opts = {
|
||||
"temperature": cfg["temperature"],
|
||||
"top_p": cfg["top_p"],
|
||||
"top_k": cfg["top_k"],
|
||||
"num_predict": cfg["max_tokens"],
|
||||
"num_batch": cfg["num_batch"],
|
||||
}
|
||||
def runner_options(cfg: dict) -> dict:
|
||||
"""Sous-ensemble d'options qui détermine l'identité du runner Ollama.
|
||||
|
||||
Ollama choisit son runner (processus de chargement du modèle) d'après
|
||||
num_ctx / num_batch / num_gpu. TOUT appel au même modèle (plan, résumé,
|
||||
agent…) doit envoyer ces mêmes valeurs, sinon Ollama recharge le modèle en
|
||||
plein milieu d'un message — la cause principale des lenteurs observées.
|
||||
"""
|
||||
opts: dict = {"num_batch": cfg["num_batch"]}
|
||||
# num_gpu n'est transmis que si l'utilisateur force explicitement un nombre
|
||||
# de couches (≥ 0). En -1 (défaut), on laisse Ollama auto-ajuster l'offload
|
||||
# GPU comme `ollama run` ; lui imposer une valeur peut le forcer sur le CPU.
|
||||
@@ -200,5 +200,16 @@ def ollama_options(cfg: dict) -> dict:
|
||||
return opts
|
||||
|
||||
|
||||
def ollama_options(cfg: dict) -> dict:
|
||||
"""Traduit la config en options de génération Ollama."""
|
||||
return {
|
||||
**runner_options(cfg),
|
||||
"temperature": cfg["temperature"],
|
||||
"top_p": cfg["top_p"],
|
||||
"top_k": cfg["top_k"],
|
||||
"num_predict": cfg["max_tokens"],
|
||||
}
|
||||
|
||||
|
||||
def enabled_tool_names(cfg: dict) -> list[str]:
|
||||
return [name for name, on in cfg["tools"].items() if on]
|
||||
Reference in new issue
Block a user