mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-22 22:48:11 +02:00
perf: fix model reload thrash and cut time-to-first-token
Ollama reloaded the chat model mid-message because plan/summary/router calls sent divergent runner options (no num_ctx/num_batch) and omitted keep_alive — the main cause of perceived slowness. - share one keep-alive httpx.AsyncClient for all Ollama calls - unify runner options (runner_options) + keep_alive on every model call, embeddings included - drop the blocking LLM routing fallback (pure lexical heuristic) - run RAG recall + plan + code-model pick in parallel inside the SSE stream, after the start event - RAG cosine scoring off the event loop; cache /api/tags 30s and nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup config fetch feat: working session menu in TopBar (switch/create/rename/delete) feat: workspace file deletion (DELETE /api/files + UI trash buttons) docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
552548e4b4
commit
cb2872c78a
23 files changed
+672
-224
No files matched your search
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
@@ -15,11 +16,21 @@ router = APIRouter(prefix="/api/system", tags=["system"])
|
||||
|
||||
_NVIDIA_SMI = shutil.which("nvidia-smi")
|
||||
|
||||
# Cache court : le front interroge /stats en continu ; relancer un sous-processus
|
||||
# nvidia-smi à chaque tick charge la machine qui héberge aussi Ollama.
|
||||
_GPU_CACHE_TTL = 5.0
|
||||
_gpu_cache: dict = {"at": 0.0, "value": None}
|
||||
|
||||
|
||||
async def _gpu_stats() -> dict | None:
|
||||
"""Utilisation GPU/VRAM via nvidia-smi ; None si absent (pas de GPU NVIDIA)."""
|
||||
"""Utilisation GPU/VRAM via nvidia-smi ; None si absent (pas de GPU NVIDIA).
|
||||
|
||||
Résultat mis en cache ~5 s pour limiter les sous-processus.
|
||||
"""
|
||||
if not _NVIDIA_SMI:
|
||||
return None
|
||||
if time.monotonic() - _gpu_cache["at"] < _GPU_CACHE_TTL:
|
||||
return _gpu_cache["value"]
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
_NVIDIA_SMI,
|
||||
@@ -31,14 +42,16 @@ async def _gpu_stats() -> dict | None:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(), timeout=3)
|
||||
line = out.decode().strip().splitlines()[0]
|
||||
util, used, total, name = (p.strip() for p in line.split(","))
|
||||
return {
|
||||
value = {
|
||||
"name": name,
|
||||
"util_pct": float(util),
|
||||
"vram_used_mb": float(used),
|
||||
"vram_total_mb": float(total),
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
value = None
|
||||
_gpu_cache.update(at=time.monotonic(), value=value)
|
||||
return value
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
|
||||
Reference in new issue
Block a user