mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-22 06:28:11 +02:00
Évite le timeout du préchargement
This commit is contained in:
1 parent
0f6abf6dde
commit
c9b6c8bb3e
4 files changed
+86
-18
No files matched your search
+6
-6
@@ -22,16 +22,16 @@ async def _warm_default_model() -> None:
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from .ollama_client import OllamaError, ollama
|
|
||||||
|
|
||||||
await asyncio.sleep(2) # laisse le service démarrer
|
await asyncio.sleep(2) # laisse le service démarrer
|
||||||
try:
|
try:
|
||||||
cfg = agent_config.get_config(settings.default_model)
|
cfg = agent_config.get_config(settings.default_model)
|
||||||
await ollama.warm(settings.default_model, cfg.get("keep_alive", "30m"))
|
models.start_model_warm(
|
||||||
logging.getLogger(__name__).info(
|
settings.default_model, cfg.get("keep_alive", "30m")
|
||||||
"Modèle %s préchargé en VRAM", settings.default_model
|
|
||||||
)
|
)
|
||||||
except (OllamaError, OSError, Exception) as exc: # best-effort
|
logging.getLogger(__name__).info(
|
||||||
|
"Préchargement du modèle %s lancé", settings.default_model
|
||||||
|
)
|
||||||
|
except Exception as exc: # best-effort
|
||||||
logging.getLogger(__name__).info(
|
logging.getLogger(__name__).info(
|
||||||
"Préchargement au démarrage ignoré : %s", exc
|
"Préchargement au démarrage ignoré : %s", exc
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -122,7 +122,10 @@ class OllamaClient:
|
|||||||
|
|
||||||
Le paramètre keep_alive fixe la durée de rétention en mémoire.
|
Le paramètre keep_alive fixe la durée de rétention en mémoire.
|
||||||
"""
|
"""
|
||||||
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
|
# Le chargement se fait désormais en tâche de fond côté API Loki. On lui
|
||||||
|
# laisse jusqu'à dix minutes pour les gros modèles ou un stockage lent.
|
||||||
|
timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0)
|
||||||
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||||
resp = await client.post(
|
resp = await client.post(
|
||||||
f"{self.host}/api/generate",
|
f"{self.host}/api/generate",
|
||||||
json={"model": model, "keep_alive": keep_alive, "stream": False},
|
json={"model": model, "keep_alive": keep_alive, "stream": False},
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
"""Routes liées à Ollama : statut de connexion, modèles, téléchargement."""
|
"""Routes liées à Ollama : statut de connexion, modèles, téléchargement."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
@@ -13,6 +14,12 @@ from ..ollama_client import ollama
|
|||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["ollama"])
|
router = APIRouter(prefix="/api", tags=["ollama"])
|
||||||
|
|
||||||
|
# État en mémoire des préchargements. Loki est lancé avec un worker unique ; ce
|
||||||
|
# suivi permet au navigateur d'interroger une route courte pendant que le long
|
||||||
|
# chargement Ollama continue sans maintenir la requête HTTP initiale ouverte.
|
||||||
|
_warm_states: dict[str, dict[str, str]] = {}
|
||||||
|
_warm_tasks: dict[str, asyncio.Task[None]] = {}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status")
|
@router.get("/status")
|
||||||
async def status() -> dict:
|
async def status() -> dict:
|
||||||
@@ -63,16 +70,47 @@ class WarmRequest(BaseModel):
|
|||||||
keep_alive: str = "30m"
|
keep_alive: str = "30m"
|
||||||
|
|
||||||
|
|
||||||
@router.post("/models/warm")
|
async def _warm_in_background(name: str, keep_alive: str) -> None:
|
||||||
async def warm_model(req: WarmRequest) -> dict:
|
|
||||||
"""Précharge un modèle en VRAM (préchargement)."""
|
|
||||||
if not req.name.strip():
|
|
||||||
raise HTTPException(400, "nom de modèle vide")
|
|
||||||
try:
|
try:
|
||||||
await ollama.warm(req.name.strip(), req.keep_alive)
|
await ollama.warm(name, keep_alive)
|
||||||
except (httpx.HTTPError, OSError) as exc:
|
_warm_states[name] = {"state": "loaded"}
|
||||||
raise HTTPException(502, f"préchargement impossible : {exc}") from exc
|
except Exception as exc:
|
||||||
return {"warmed": req.name.strip()}
|
_warm_states[name] = {
|
||||||
|
"state": "error",
|
||||||
|
"error": f"préchargement impossible : {str(exc)[:500]}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def start_model_warm(name: str, keep_alive: str) -> None:
|
||||||
|
"""Démarre au plus une tâche de préchargement par modèle."""
|
||||||
|
current = _warm_tasks.get(name)
|
||||||
|
if current and not current.done():
|
||||||
|
return
|
||||||
|
_warm_states[name] = {"state": "loading"}
|
||||||
|
task = asyncio.create_task(_warm_in_background(name, keep_alive))
|
||||||
|
_warm_tasks[name] = task
|
||||||
|
|
||||||
|
def forget(done: asyncio.Task[None]) -> None:
|
||||||
|
if _warm_tasks.get(name) is done:
|
||||||
|
_warm_tasks.pop(name, None)
|
||||||
|
|
||||||
|
task.add_done_callback(forget)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/models/warm", status_code=202)
|
||||||
|
async def warm_model(req: WarmRequest) -> dict:
|
||||||
|
"""Démarre le préchargement sans exposer sa durée au reverse proxy."""
|
||||||
|
name = req.name.strip()
|
||||||
|
if not name:
|
||||||
|
raise HTTPException(400, "nom de modèle vide")
|
||||||
|
start_model_warm(name, req.keep_alive)
|
||||||
|
return {"warming": name, "state": _warm_states[name]["state"]}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/models/warm/status")
|
||||||
|
async def warm_status(name: str = Query(min_length=1)) -> dict:
|
||||||
|
"""État court du préchargement : idle, loading, loaded ou error."""
|
||||||
|
return _warm_states.get(name.strip(), {"state": "idle"})
|
||||||
|
|
||||||
|
|
||||||
@router.get("/models/loaded")
|
@router.get("/models/loaded")
|
||||||
|
|||||||
@@ -56,7 +56,17 @@ async function apiError(res: Response, fallback: string): Promise<Error> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Précharge un modèle en mémoire et remonte toute erreur à l'interface. */
|
interface WarmStatus {
|
||||||
|
state: "idle" | "loading" | "loaded" | "error";
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lance le préchargement en arrière-plan puis suit son état. Chaque requête
|
||||||
|
* reste courte afin qu'un reverse proxy ne puisse plus interrompre le warm-up.
|
||||||
|
*/
|
||||||
export async function warmModel(name: string, keepAlive = "30m"): Promise<void> {
|
export async function warmModel(name: string, keepAlive = "30m"): Promise<void> {
|
||||||
const res = await fetch("/api/models/warm", {
|
const res = await fetch("/api/models/warm", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
@@ -66,6 +76,23 @@ export async function warmModel(name: string, keepAlive = "30m"): Promise<void>
|
|||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
throw await apiError(res, `préchargement refusé (${res.status})`);
|
throw await apiError(res, `préchargement refusé (${res.status})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt < 300; attempt++) {
|
||||||
|
const statusRes = await fetch(
|
||||||
|
`/api/models/warm/status?name=${encodeURIComponent(name)}`,
|
||||||
|
{ cache: "no-store" }
|
||||||
|
);
|
||||||
|
if (!statusRes.ok) {
|
||||||
|
throw await apiError(statusRes, `suivi du préchargement refusé (${statusRes.status})`);
|
||||||
|
}
|
||||||
|
const status = (await statusRes.json()) as WarmStatus;
|
||||||
|
if (status.state === "loaded") return;
|
||||||
|
if (status.state === "error") {
|
||||||
|
throw new Error(status.error ?? "préchargement impossible");
|
||||||
|
}
|
||||||
|
await wait(2000);
|
||||||
|
}
|
||||||
|
throw new Error("préchargement toujours en cours après 10 minutes");
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getLoadedModels(): Promise<LoadedModel[]> {
|
export async function getLoadedModels(): Promise<LoadedModel[]> {
|
||||||
|
|||||||
Reference in new issue
Block a user