mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-23 06:58:14 +02:00
perf: fix model reload thrash and cut time-to-first-token
Ollama reloaded the chat model mid-message because plan/summary/router calls sent divergent runner options (no num_ctx/num_batch) and omitted keep_alive — the main cause of perceived slowness. - share one keep-alive httpx.AsyncClient for all Ollama calls - unify runner options (runner_options) + keep_alive on every model call, embeddings included - drop the blocking LLM routing fallback (pure lexical heuristic) - run RAG recall + plan + code-model pick in parallel inside the SSE stream, after the start event - RAG cosine scoring off the event loop; cache /api/tags 30s and nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup config fetch feat: working session menu in TopBar (switch/create/rename/delete) feat: workspace file deletion (DELETE /api/files + UI trash buttons) docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
552548e4b4
commit
cb2872c78a
23 files changed
+672
-224
No files matched your search
+38
-5
@@ -46,13 +46,28 @@ def needs_plan(message: str) -> bool:
|
||||
return len(message) > 240 or connectors >= 2
|
||||
|
||||
|
||||
async def _ask(model: str, system: str, user: str, *, num_predict: int) -> str:
|
||||
async def _ask(
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
num_predict: int,
|
||||
options: dict | None = None,
|
||||
keep_alive: str | None = None,
|
||||
) -> str:
|
||||
"""Appel court au modèle.
|
||||
|
||||
``options`` doit contenir les options runner (num_ctx, num_batch…) du chat
|
||||
principal : un appel avec des options divergentes force Ollama à recharger
|
||||
le modèle en plein message.
|
||||
"""
|
||||
text = ""
|
||||
async for chunk in ollama.chat(
|
||||
model,
|
||||
[{"role": "system", "content": system},
|
||||
{"role": "user", "content": user}],
|
||||
options={"temperature": 0.2, "num_predict": num_predict},
|
||||
options={**(options or {}), "temperature": 0.2, "num_predict": num_predict},
|
||||
keep_alive=keep_alive,
|
||||
stream=True,
|
||||
):
|
||||
text += chunk.get("message", {}).get("content", "")
|
||||
@@ -61,10 +76,19 @@ async def _ask(model: str, system: str, user: str, *, num_predict: int) -> str:
|
||||
return text.strip()
|
||||
|
||||
|
||||
async def make_plan(model: str, message: str) -> list[str]:
|
||||
async def make_plan(
|
||||
model: str,
|
||||
message: str,
|
||||
*,
|
||||
options: dict | None = None,
|
||||
keep_alive: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Renvoie la liste des étapes (vide si échec — jamais bloquant)."""
|
||||
try:
|
||||
raw = await _ask(model, _PLAN_PROMPT, message[:1200], num_predict=220)
|
||||
raw = await _ask(
|
||||
model, _PLAN_PROMPT, message[:1200], num_predict=220,
|
||||
options=options, keep_alive=keep_alive,
|
||||
)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
logger.warning("Plan impossible : %s", exc)
|
||||
return []
|
||||
@@ -77,7 +101,14 @@ async def make_plan(model: str, message: str) -> list[str]:
|
||||
return steps[:5] if len(steps) >= 2 else []
|
||||
|
||||
|
||||
async def self_review(model: str, request: str, answer: str) -> str | None:
|
||||
async def self_review(
|
||||
model: str,
|
||||
request: str,
|
||||
answer: str,
|
||||
*,
|
||||
options: dict | None = None,
|
||||
keep_alive: str | None = None,
|
||||
) -> str | None:
|
||||
"""Critique puis révise la réponse. None si rien à corriger / échec."""
|
||||
if len(answer) < 80:
|
||||
return None
|
||||
@@ -87,6 +118,7 @@ async def self_review(model: str, request: str, answer: str) -> str | None:
|
||||
_CRITIQUE_PROMPT,
|
||||
f"Demande :\n{request[:800]}\n\nRéponse :\n{answer[:2500]}",
|
||||
num_predict=180,
|
||||
options=options, keep_alive=keep_alive,
|
||||
)
|
||||
if not critique or "PARFAIT" in critique.upper()[:40]:
|
||||
return None
|
||||
@@ -97,6 +129,7 @@ async def self_review(model: str, request: str, answer: str) -> str | None:
|
||||
f"Demande :\n{request[:800]}\n\nRéponse initiale :\n{answer[:2500]}"
|
||||
f"\n\nDéfauts :\n{critique[:600]}",
|
||||
num_predict=1500,
|
||||
options=options, keep_alive=keep_alive,
|
||||
)
|
||||
# Garde-fou : une révision vide ou minuscule ne remplace rien.
|
||||
return revised if len(revised) > len(answer) // 3 else None
|
||||
|
||||
Reference in new issue
Block a user