perf: fix model reload thrash and cut time-to-first-token

Ollama reloaded the chat model mid-message because plan/summary/router
calls sent divergent runner options (no num_ctx/num_batch) and omitted
keep_alive — the main cause of perceived slowness.

- share one keep-alive httpx.AsyncClient for all Ollama calls
- unify runner options (runner_options) + keep_alive on every model
  call, embeddings included
- drop the blocking LLM routing fallback (pure lexical heuristic)
- run RAG recall + plan + code-model pick in parallel inside the SSE
  stream, after the start event
- RAG cosine scoring off the event loop; cache /api/tags 30s and
  nvidia-smi 5s; frontend polls 2s->5s, warm poll backoff, dedup
  config fetch

feat: working session menu in TopBar (switch/create/rename/delete)
feat: workspace file deletion (DELETE /api/files + UI trash buttons)
docs: recommended Ollama env vars (KEEP_ALIVE, MAX_LOADED_MODELS...)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MichaelandClaude Opus 4.8 committed 2026-07-18 13:58:18 +02:00
1 parent 552548e4b4
commit cb2872c78a
23 files changed
+672 -224

No files matched your search

+38 -5
View File
@@ -46,13 +46,28 @@ def needs_plan(message: str) -> bool:
return len(message) > 240 or connectors >= 2
async def _ask(model: str, system: str, user: str, *, num_predict: int) -> str:
async def _ask(
model: str,
system: str,
user: str,
*,
num_predict: int,
options: dict | None = None,
keep_alive: str | None = None,
) -> str:
"""Appel court au modèle.
``options`` doit contenir les options runner (num_ctx, num_batch…) du chat
principal : un appel avec des options divergentes force Ollama à recharger
le modèle en plein message.
"""
text = ""
async for chunk in ollama.chat(
model,
[{"role": "system", "content": system},
{"role": "user", "content": user}],
options={"temperature": 0.2, "num_predict": num_predict},
options={**(options or {}), "temperature": 0.2, "num_predict": num_predict},
keep_alive=keep_alive,
stream=True,
):
text += chunk.get("message", {}).get("content", "")
@@ -61,10 +76,19 @@ async def _ask(model: str, system: str, user: str, *, num_predict: int) -> str:
return text.strip()
async def make_plan(model: str, message: str) -> list[str]:
async def make_plan(
model: str,
message: str,
*,
options: dict | None = None,
keep_alive: str | None = None,
) -> list[str]:
"""Renvoie la liste des étapes (vide si échec — jamais bloquant)."""
try:
raw = await _ask(model, _PLAN_PROMPT, message[:1200], num_predict=220)
raw = await _ask(
model, _PLAN_PROMPT, message[:1200], num_predict=220,
options=options, keep_alive=keep_alive,
)
except (httpx.HTTPError, OSError) as exc:
logger.warning("Plan impossible : %s", exc)
return []
@@ -77,7 +101,14 @@ async def make_plan(model: str, message: str) -> list[str]:
return steps[:5] if len(steps) >= 2 else []
async def self_review(model: str, request: str, answer: str) -> str | None:
async def self_review(
model: str,
request: str,
answer: str,
*,
options: dict | None = None,
keep_alive: str | None = None,
) -> str | None:
"""Critique puis révise la réponse. None si rien à corriger / échec."""
if len(answer) < 80:
return None
@@ -87,6 +118,7 @@ async def self_review(model: str, request: str, answer: str) -> str | None:
_CRITIQUE_PROMPT,
f"Demande :\n{request[:800]}\n\nRéponse :\n{answer[:2500]}",
num_predict=180,
options=options, keep_alive=keep_alive,
)
if not critique or "PARFAIT" in critique.upper()[:40]:
return None
@@ -97,6 +129,7 @@ async def self_review(model: str, request: str, answer: str) -> str | None:
f"Demande :\n{request[:800]}\n\nRéponse initiale :\n{answer[:2500]}"
f"\n\nDéfauts :\n{critique[:600]}",
num_predict=1500,
options=options, keep_alive=keep_alive,
)
# Garde-fou : une révision vide ou minuscule ne remplace rien.
return revised if len(revised) > len(answer) // 3 else None