mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-17 04:10:04 +02:00
Lot 14 : ligne de commande du moteur composée en Rust (LOKI_SERVE=rust)
loki-rs engine serve fait ce que faisait loki serve : lire la configuration (par l'API interne du Go), sonder le monde (aide du moteur, --version, --list-devices, GGUF, RAM, VRAM, cgroup, cpuset, variables LLAMA_ARG_*/GGML_*), composer la ligne de llama-server, vérifier le port, garder la commande pour l'interface, puis exec. Le superviseur le lance à la place de loki serve quand LOKI_SERVE=rust (expérimental ; le Go reste le défaut). serve/ porte backend_serve*.go fonction par fonction (couches GPU, --fit et sa marge, threads, files CUDA, second slot, mode tensor, cache de prompts, points de reprise, décodage spéculatif, fidélité, LOAD_GUARD, SLOT_PERSIST), toute la décision restant pure. Strata, l'essai de l'optimiseur et un lancement sans API interne passent toujours par loki serve (Go). La façade donne au Go l'adresse de son API interne (LOKI_GO_ADDR), héritée par le superviseur. Parité : parity_serve_test.go tire du Go 1 872 lancements (12 machines × 112 presets × 3 aides de moteur × 13 jeux de variables) et les sondes pures, rejoués argument par argument par tests/serve.rs ; deux essais de bout en bout lancent le vrai binaire contre un faux Go et un faux moteur. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016tcGfJWefW5kUGoC2wL8cv
This commit is contained in:
22 files changed
+66931
-5
No files matched your search
@@ -41,6 +41,10 @@ impl Paths {
|
||||
pub struct Opts {
|
||||
/// Binaire Go à lancer en `serve`.
|
||||
pub go_bin: PathBuf,
|
||||
/// Commande qui compose la ligne du moteur puis lui cède la place :
|
||||
/// `loki serve` (Go) par défaut, `loki-rs engine serve` quand
|
||||
/// `LOKI_SERVE=rust` (lot 14).
|
||||
pub serve: (PathBuf, Vec<String>),
|
||||
/// Délai avant de juger que le démarrage a tenu (2 s comme le Go).
|
||||
pub check_delay: Duration,
|
||||
/// SIGTERM → SIGKILL (4 s comme le Go).
|
||||
@@ -52,6 +56,7 @@ pub struct Opts {
|
||||
impl Opts {
|
||||
pub fn new(go_bin: PathBuf) -> Self {
|
||||
Opts {
|
||||
serve: (go_bin.clone(), vec!["serve".into()]),
|
||||
go_bin,
|
||||
check_delay: Duration::from_secs(2),
|
||||
grace: Duration::from_secs(4),
|
||||
@@ -124,8 +129,8 @@ mod imp {
|
||||
.open(paths.log())
|
||||
.map_err(|e| format!("ouverture du log {}: {e}", paths.log().display()))?;
|
||||
let log2 = log.try_clone().map_err(|e| e.to_string())?;
|
||||
let mut cmd = Command::new(&opts.go_bin);
|
||||
cmd.arg("serve")
|
||||
let mut cmd = Command::new(&opts.serve.0);
|
||||
cmd.args(&opts.serve.1)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(log)
|
||||
.stderr(log2)
|
||||
|
||||
@@ -30,6 +30,7 @@ pub mod paths;
|
||||
pub mod presets;
|
||||
pub mod proxy;
|
||||
pub mod push;
|
||||
pub mod serve;
|
||||
pub mod settings;
|
||||
pub mod store;
|
||||
pub mod sysinfo;
|
||||
|
||||
@@ -21,8 +21,17 @@ fn main() -> ExitCode {
|
||||
"engine" => {
|
||||
let action = std::env::args().nth(2).unwrap_or_default();
|
||||
let go_bin = child::go_bin(std::env::var("LOKI_GO_BIN").ok(), &exe());
|
||||
let opts = engine::Opts::new(go_bin);
|
||||
match engine::run(&action, &engine::Paths::from_env(), &opts) {
|
||||
let paths = engine::Paths::from_env();
|
||||
// La ligne de commande du moteur composée ici (lot 14) : lancée
|
||||
// par le superviseur quand LOKI_SERVE=rust.
|
||||
if action == "serve" {
|
||||
return loki_rs::serve::run::run(&go_bin, &paths.home);
|
||||
}
|
||||
let mut opts = engine::Opts::new(go_bin);
|
||||
if held_by_facade("LOKI_SERVE") {
|
||||
opts.serve = (exe(), vec!["engine".into(), "serve".into()]);
|
||||
}
|
||||
match engine::run(&action, &paths, &opts) {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(e) => {
|
||||
// Dernière ligne de stderr : le Go en fait son message d'erreur.
|
||||
@@ -136,7 +145,12 @@ fn serve() -> ExitCode {
|
||||
// LOKI_MEM=rust : la mémoire (pages, index, trackers, mode mémoire,
|
||||
// interrupteurs du mode agent).
|
||||
let mem_rust = held_by_facade("LOKI_MEM");
|
||||
let mut env = vec![(auth::INTERNAL_ENV.to_string(), token.clone())];
|
||||
let mut env = vec![
|
||||
(auth::INTERNAL_ENV.to_string(), token.clone()),
|
||||
// Adresse de l'API interne du Go, héritée par ses enfants :
|
||||
// « loki-rs engine serve » (LOKI_SERVE=rust) y lit la configuration.
|
||||
("LOKI_GO_ADDR".to_string(), backend.to_string()),
|
||||
];
|
||||
let held = [
|
||||
("LOKI_CHAT", chat_rust),
|
||||
("LOKI_MODELS", models_rust),
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
//! Cache de prompts en RAM et dossier des slots (`backend_serve_cache.go`) :
|
||||
//! --cache-ram agrandi quand le besoin se mesure et que la RAM est libre,
|
||||
//! jamais sous le défaut du moteur ; --slot-save-path seulement pour un
|
||||
//! moteur fermé aux autres.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, atoi, effective_kv_types, env_truthy, flag_value, flag_values, get, has_any_flag,
|
||||
kv_offloaded, slot_persist_plan, trim,
|
||||
};
|
||||
use crate::gguf::GgufMeta;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// `engineCacheRAMDefault` : cache_ram_mib par défaut de llama.cpp.
|
||||
pub const ENGINE_CACHE_RAM_DEFAULT: i64 = 8192;
|
||||
/// `cacheArgEnv`.
|
||||
pub const CACHE_ARG_ENV: &[&str] = &["LLAMA_ARG_CACHE_RAM"];
|
||||
|
||||
/// `cgroupMemMiB` : (limite, consommation) du conteneur en Mio ; 0 = aucune.
|
||||
pub fn cgroup_mem_mib(root: &Path) -> (i64, i64) {
|
||||
let read = |p: &str| -> i64 {
|
||||
let Ok(b) = std::fs::read(root.join(p)) else {
|
||||
return 0;
|
||||
};
|
||||
match atoi(trim(&String::from_utf8_lossy(&b))) {
|
||||
Some(n) if n > 0 && n < 1 << 60 => n >> 20,
|
||||
_ => 0,
|
||||
}
|
||||
};
|
||||
let limit = read("sys/fs/cgroup/memory.max");
|
||||
if limit > 0 {
|
||||
return (limit, read("sys/fs/cgroup/memory.current"));
|
||||
}
|
||||
let limit = read("sys/fs/cgroup/memory/memory.limit_in_bytes");
|
||||
if limit > 0 {
|
||||
return (limit, read("sys/fs/cgroup/memory/memory.usage_in_bytes"));
|
||||
}
|
||||
(0, 0)
|
||||
}
|
||||
|
||||
/// `vramFromSMI` : VRAM des cartes choisies par CUDA_VISIBLE_DEVICES.
|
||||
pub fn vram_from_smi(out: &str, cvd: &str, pci_order: bool) -> i64 {
|
||||
let mut by_idx: HashMap<i64, i64> = HashMap::new();
|
||||
let mut min_mib = 0i64;
|
||||
for line in trim(out).split('\n') {
|
||||
let f: Vec<&str> = line.split(',').collect();
|
||||
if f.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
match (atoi(trim(f[0])), atoi(trim(f[1]))) {
|
||||
(Some(i), Some(m)) if m > 0 => {
|
||||
by_idx.insert(i, m);
|
||||
if min_mib == 0 || m < min_mib {
|
||||
min_mib = m;
|
||||
}
|
||||
}
|
||||
_ => return 0,
|
||||
}
|
||||
}
|
||||
if by_idx.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let cvd = trim(cvd);
|
||||
if cvd.is_empty() {
|
||||
return by_idx.values().sum();
|
||||
}
|
||||
let (mut sum, mut n) = (0i64, 0i64);
|
||||
for d in cvd.split(',') {
|
||||
let d = trim(d);
|
||||
if d.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some(i) = atoi(d) else { return 0 };
|
||||
let Some(m) = by_idx.get(&i) else { return 0 };
|
||||
sum += m;
|
||||
n += 1;
|
||||
}
|
||||
if !pci_order {
|
||||
return min_mib * n;
|
||||
}
|
||||
sum
|
||||
}
|
||||
|
||||
/// `kvTypeBytes` : octets par élément d'un type de cache.
|
||||
pub fn kv_type_bytes(t: &str) -> f64 {
|
||||
match trim(t).to_lowercase().as_str() {
|
||||
"f32" => 4.0,
|
||||
"q8_0" => 34.0 / 32.0,
|
||||
"q5_1" => 24.0 / 32.0,
|
||||
"q5_0" => 22.0 / 32.0,
|
||||
"q4_1" => 20.0 / 32.0,
|
||||
"q4_0" | "iq4_nl" => 18.0 / 32.0,
|
||||
_ => 2.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// `ggufStateBytes` : (octets de cache KV par jeton, octets d'état
|
||||
/// récurrent).
|
||||
pub fn gguf_state_bytes(g: &GgufMeta, kt: &str, vt: &str) -> (f64, i64) {
|
||||
let (mut kl, mut vl) = (g.key_len, g.val_len);
|
||||
if g.embedding_length > 0 && g.head_count > 0 {
|
||||
if kl == 0 {
|
||||
kl = g.embedding_length / g.head_count;
|
||||
}
|
||||
if vl == 0 {
|
||||
vl = g.embedding_length / g.head_count;
|
||||
}
|
||||
}
|
||||
if kl == 0 || vl == 0 || g.block_count == 0 {
|
||||
return (0.0, 0);
|
||||
}
|
||||
let (mut heads, mut rec_layers) = (0i64, 0i64);
|
||||
match g.head_count_kv_layers.as_deref() {
|
||||
Some(layers) if !layers.is_empty() => {
|
||||
for &h in layers {
|
||||
heads += h;
|
||||
if h == 0 && g.hybrid {
|
||||
rec_layers += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ if g.hybrid && g.full_attn_interval > 0 => {
|
||||
let attn = g.block_count / g.full_attn_interval;
|
||||
heads = attn * g.head_count_kv;
|
||||
rec_layers = g.block_count - attn;
|
||||
}
|
||||
_ => {
|
||||
heads = g.block_count * g.head_count_kv;
|
||||
if heads == 0 && g.hybrid {
|
||||
rec_layers = g.block_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
let per_token = heads as f64 * (kl as f64 * kv_type_bytes(kt) + vl as f64 * kv_type_bytes(vt));
|
||||
let mut recurrent = 0i64;
|
||||
if rec_layers > 0 {
|
||||
let conv = if g.ssm_conv > 1 {
|
||||
(g.ssm_conv - 1) * (g.ssm_inner + 2 * g.ssm_groups * g.ssm_state)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
recurrent = rec_layers * (conv + g.ssm_state * g.ssm_inner) * 4;
|
||||
}
|
||||
(per_token, recurrent)
|
||||
}
|
||||
|
||||
/// `serveCtx` : contexte que le moteur ouvrira (0 = celui d'entraînement).
|
||||
pub fn serve_ctx(cfg: &Cfg, extra: &[String], g: Option<&GgufMeta>) -> i64 {
|
||||
let mut sv = flag_value(extra, &["-c", "--ctx-size"]);
|
||||
if sv.is_empty() {
|
||||
sv = trim(get(cfg, "CTX")).to_string();
|
||||
}
|
||||
if sv.is_empty() {
|
||||
sv = "32768".into();
|
||||
}
|
||||
let mut n = match atoi(&sv) {
|
||||
Some(n) if n >= 0 => n,
|
||||
_ => return 0,
|
||||
};
|
||||
if n == 0
|
||||
&& let Some(g) = g
|
||||
{
|
||||
n = g.context_length;
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
/// `cpuWeights` : pourquoi des poids (ou le cache KV) vivent en RAM hôte.
|
||||
pub fn cpu_weights(
|
||||
cfg: &Cfg,
|
||||
extra: &[String],
|
||||
arg_env: &HashMap<String, String>,
|
||||
blocks: i64,
|
||||
) -> String {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
let mut ot = flag_values(extra, &["-ot", "--override-tensor"]);
|
||||
if !env("LLAMA_ARG_OVERRIDE_TENSOR").is_empty() {
|
||||
ot.push(env("LLAMA_ARG_OVERRIDE_TENSOR").into());
|
||||
}
|
||||
for v in &ot {
|
||||
for el in v.split(',') {
|
||||
if let Some((_, buft)) = el.split_once('=')
|
||||
&& trim(buft).to_uppercase().starts_with("CPU")
|
||||
{
|
||||
return "tenseurs sur CPU (-ot)".into();
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_any_flag(extra, &["--cpu-moe", "-cmoe"]) || env_truthy(env("LLAMA_ARG_CPU_MOE")) {
|
||||
return "experts sur CPU".into();
|
||||
}
|
||||
for (e, short, long) in [
|
||||
("LLAMA_ARG_N_CPU_MOE", "-ncmoe", "--n-cpu-moe"),
|
||||
("LLAMA_ARG_N_CPU_FFN", "-ncffn", "--n-cpu-ffn"),
|
||||
] {
|
||||
let mut vals = flag_values(extra, &[short, long]);
|
||||
vals.push(env(e).to_string());
|
||||
for v in vals {
|
||||
let v = trim(&v);
|
||||
if !v.is_empty() && v != "0" {
|
||||
return format!("couches {long} sur CPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
if !kv_offloaded(extra, arg_env) {
|
||||
return "cache KV sur CPU".into();
|
||||
}
|
||||
let mut ngl = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if ngl.is_empty() {
|
||||
ngl = trim(get(cfg, "NGL")).to_string();
|
||||
if ngl.eq_ignore_ascii_case("auto") {
|
||||
ngl = env("LLAMA_ARG_N_GPU_LAYERS").into();
|
||||
}
|
||||
}
|
||||
if let Some(n) = atoi(trim(&ngl))
|
||||
&& n != 999
|
||||
&& (blocks == 0 || n < blocks + 1)
|
||||
{
|
||||
return format!("couches GPU limitées à {n}");
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `cacheRAMAuto` : taille du cache de prompts (0 = défaut du moteur).
|
||||
pub fn cache_ram_auto(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (i64, String) {
|
||||
if si.vram_mib <= 0 {
|
||||
return (0, "VRAM NVIDIA inconnue".into());
|
||||
}
|
||||
let Some(g) = si.gguf.as_ref() else {
|
||||
return (0, "métadonnées GGUF illisibles".into());
|
||||
};
|
||||
if si.ram_mib <= 0 || si.ram_avail_mib <= 0 {
|
||||
return (0, "RAM inconnue".into());
|
||||
}
|
||||
if si.model_bytes <= 0 {
|
||||
return (0, "taille du modèle inconnue".into());
|
||||
}
|
||||
let r = cpu_weights(cfg, extra, &si.arg_env, g.block_count);
|
||||
if !r.is_empty() {
|
||||
return (0, r);
|
||||
}
|
||||
let (kt, vt, _) = effective_kv_types(cfg, extra, &si.arg_env);
|
||||
let (per_tok, rec) = gguf_state_bytes(g, &kt, &vt);
|
||||
let ctx = serve_ctx(cfg, extra, Some(g));
|
||||
if per_tok <= 0.0 || ctx <= 0 {
|
||||
return (0, "taille d'état inconnue".into());
|
||||
}
|
||||
let mib = 1048576.0;
|
||||
let kv_bytes = per_tok * ctx as f64;
|
||||
if si.model_bytes as f64 + kv_bytes > 0.9 * si.vram_mib as f64 * mib {
|
||||
return (
|
||||
0,
|
||||
"modèle trop gros pour la VRAM : des poids restent en RAM".into(),
|
||||
);
|
||||
}
|
||||
let mut state = kv_bytes;
|
||||
if rec > 0 {
|
||||
state += rec as f64 * ((32i64.min(ctx / 8192 + 1) + 1) as f64);
|
||||
}
|
||||
let need = (2.5 * state / mib).ceil() as i64;
|
||||
let ceiling = (si.ram_mib * 30 / 100).min(si.ram_avail_mib / 2);
|
||||
if ceiling <= ENGINE_CACHE_RAM_DEFAULT {
|
||||
return (0, format!("RAM libre trop juste (plafond {ceiling} Mio)"));
|
||||
}
|
||||
if need <= ENGINE_CACHE_RAM_DEFAULT {
|
||||
return (0, format!("le défaut suffit (besoin estimé {need} Mio)"));
|
||||
}
|
||||
(
|
||||
need.min(ceiling),
|
||||
format!("besoin estimé {need} Mio, plafond {ceiling} Mio"),
|
||||
)
|
||||
}
|
||||
|
||||
/// `cacheRAMSetting` : (valeur manuelle, auto, lisible).
|
||||
pub fn cache_ram_setting(cfg: &Cfg) -> (String, bool, bool) {
|
||||
let v = trim(get(cfg, "CACHE_RAM"));
|
||||
if v.is_empty() || v.eq_ignore_ascii_case("auto") {
|
||||
return (String::new(), true, true);
|
||||
}
|
||||
match atoi(v) {
|
||||
Some(n) if n >= -1 => (n.to_string(), false, true),
|
||||
_ => (String::new(), false, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// `cacheRAMArgs`.
|
||||
pub fn cache_ram_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (Vec<String>, Vec<String>) {
|
||||
let (manual, auto, ok) = cache_ram_setting(cfg);
|
||||
if !ok {
|
||||
return (
|
||||
Vec::new(),
|
||||
vec![format!(
|
||||
"CACHE_RAM={} illisible (un nombre de Mio, -1, 0 ou auto) : ignoré",
|
||||
get(cfg, "CACHE_RAM")
|
||||
)],
|
||||
);
|
||||
}
|
||||
let mut notes = Vec::new();
|
||||
if !si.help.contains("--cache-ram") {
|
||||
if !auto {
|
||||
notes.push("ce moteur ne connaît pas --cache-ram : CACHE_RAM ignoré".into());
|
||||
}
|
||||
return (Vec::new(), notes);
|
||||
}
|
||||
if has_any_flag(extra, &["-cram", "--cache-ram"]) {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
if !si.env("LLAMA_ARG_CACHE_RAM").is_empty() {
|
||||
if !auto {
|
||||
notes.push("LLAMA_ARG_CACHE_RAM est posée : CACHE_RAM ignoré".into());
|
||||
}
|
||||
return (Vec::new(), notes);
|
||||
}
|
||||
if !auto {
|
||||
return (vec!["--cache-ram".into(), manual], Vec::new());
|
||||
}
|
||||
let (n, why) = cache_ram_auto(cfg, extra, si);
|
||||
if n <= 0 {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
(
|
||||
vec!["--cache-ram".into(), n.to_string()],
|
||||
vec![format!(
|
||||
"cache de prompts : --cache-ram {n} (RAM hôte, copie exacte de l'état, jamais la VRAM ; {why}). CACHE_RAM=<Mio> pour fixer, 0 pour couper."
|
||||
)],
|
||||
)
|
||||
}
|
||||
|
||||
/// `cacheRAMOff`.
|
||||
pub fn cache_ram_off(cfg: &Cfg, extra: &[String], arg_env: &HashMap<String, String>) -> bool {
|
||||
let mut v = flag_value(extra, &["-cram", "--cache-ram"]);
|
||||
if v.is_empty() {
|
||||
v = arg_env
|
||||
.get("LLAMA_ARG_CACHE_RAM")
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
}
|
||||
if v.is_empty() {
|
||||
v = cache_ram_setting(cfg).0;
|
||||
}
|
||||
trim(&v) == "0"
|
||||
}
|
||||
|
||||
/// `cacheIsolationOn`.
|
||||
pub fn cache_isolation_on(cfg: &Cfg) -> bool {
|
||||
!trim(get(cfg, "CACHE_ISOLATE")).eq_ignore_ascii_case("off")
|
||||
}
|
||||
|
||||
/// `loopbackHost`.
|
||||
pub fn loopback_host(h: &str) -> bool {
|
||||
matches!(
|
||||
trim(h).trim_matches(['[', ']']),
|
||||
"127.0.0.1" | "::1" | "localhost"
|
||||
)
|
||||
}
|
||||
|
||||
/// `slotSaveArgs`.
|
||||
pub fn slot_save_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> Vec<String> {
|
||||
if si.slot_dir.is_empty()
|
||||
|| !si.help.contains("--slot-save-path")
|
||||
|| has_any_flag(extra, &["--slot-save-path"])
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
let (persist, _) = slot_persist_plan(cfg, extra, si);
|
||||
if !persist && (!cache_isolation_on(cfg) || cache_ram_off(cfg, extra, &si.arg_env)) {
|
||||
return Vec::new();
|
||||
}
|
||||
if !super::slot_save_safe(cfg, extra, si) {
|
||||
return Vec::new();
|
||||
}
|
||||
vec!["--slot-save-path".into(), si.slot_dir.clone()]
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Points de reprise des modèles hybrides et conservation du raisonnement
|
||||
//! (`backend_serve_ckpt.go`) : --ctx-checkpoints, --checkpoint-min-step,
|
||||
//! --reasoning-preserve. Le build n'est cru que pour un moteur officiel.
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, atoi, flag_value, flag_values, get, gguf_state_bytes, has_any_flag, trim,
|
||||
};
|
||||
use crate::gguf::GgufMeta;
|
||||
|
||||
pub const ENGINE_MIN_RECOMMENDED: i64 = 10864;
|
||||
pub const ENGINE_BUILD_FLOOR: i64 = 5000;
|
||||
pub const ENGINE_CKPT_DEFAULT: i64 = 32;
|
||||
pub const CKPT_AUTO_MIN_STEP: i64 = 2048;
|
||||
pub const CKPT_FALLBACK_MIB: i64 = 200;
|
||||
|
||||
/// `ckptArgEnv`.
|
||||
pub const CKPT_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_CTX_CHECKPOINTS",
|
||||
"LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT",
|
||||
"LLAMA_ARG_REASONING_PRESERVE",
|
||||
"LLAMA_ARG_CHAT_TEMPLATE_KWARGS",
|
||||
];
|
||||
|
||||
/// `ggufHybrid` : couches récurrentes (métadonnées, ou architecture connue).
|
||||
pub fn gguf_hybrid(g: Option<&GgufMeta>) -> bool {
|
||||
g.is_some_and(|g| {
|
||||
g.hybrid
|
||||
|| matches!(
|
||||
g.arch.to_lowercase().as_str(),
|
||||
"qwen35" | "qwen35moe" | "qwen3next" | "qwen4exp"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// `engineBuildTrust` : build cru seulement pour un moteur officiel.
|
||||
pub fn engine_build_trust(source: &str, build: i64) -> i64 {
|
||||
if source == "custom" || build < ENGINE_BUILD_FLOOR {
|
||||
0
|
||||
} else {
|
||||
build
|
||||
}
|
||||
}
|
||||
|
||||
/// `engineBuildNotice`.
|
||||
pub fn engine_build_notice(build: i64) -> String {
|
||||
if build <= 0 || build >= ENGINE_MIN_RECOMMENDED {
|
||||
return String::new();
|
||||
}
|
||||
format!(
|
||||
"moteur b{build} : les points de reprise des modèles hybrides (Qwen3.5/3.6, Qwen3-Next…) y sont évincés trop tôt — jusqu'à ~8 k jetons recalculés à chaque reprise. Corrigé à partir de b{ENGINE_MIN_RECOMMENDED}."
|
||||
)
|
||||
}
|
||||
|
||||
/// `ckptCount` : points de reprise par slot (-1 = illisible).
|
||||
pub fn ckpt_count(cfg: &Cfg, extra: &[String], si: &SysInfo) -> i64 {
|
||||
let mut v = flag_value(extra, &["-ctxcp", "--ctx-checkpoints", "--swa-checkpoints"]);
|
||||
if v.is_empty() {
|
||||
v = si.env("LLAMA_ARG_CTX_CHECKPOINTS").into();
|
||||
}
|
||||
if v.is_empty() {
|
||||
return match atoi(trim(get(cfg, "CTX_CHECKPOINTS"))) {
|
||||
Some(n) if n >= 0 => n,
|
||||
_ => ENGINE_CKPT_DEFAULT,
|
||||
};
|
||||
}
|
||||
match atoi(trim(&v)) {
|
||||
Some(n) if n >= 0 => n,
|
||||
_ => -1,
|
||||
}
|
||||
}
|
||||
|
||||
/// `ckptSlots`.
|
||||
pub fn ckpt_slots(cfg: &Cfg, extra: &[String]) -> i64 {
|
||||
let mut v = flag_value(extra, &["-np", "--parallel"]);
|
||||
if v.is_empty() {
|
||||
v = trim(get(cfg, "PARALLEL")).to_string();
|
||||
}
|
||||
if v.is_empty() {
|
||||
return 1;
|
||||
}
|
||||
match atoi(trim(&v)) {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => 4,
|
||||
}
|
||||
}
|
||||
|
||||
/// `ckptMiB`.
|
||||
pub fn ckpt_mib(g: Option<&GgufMeta>) -> i64 {
|
||||
if let Some(g) = g {
|
||||
let (_, rec) = gguf_state_bytes(g, "", "");
|
||||
if rec > 0 {
|
||||
return (rec + (1 << 20) - 1) >> 20;
|
||||
}
|
||||
}
|
||||
CKPT_FALLBACK_MIB
|
||||
}
|
||||
|
||||
/// `ramTight` : le modèle occupe plus de 70 % de la RAM.
|
||||
pub fn ram_tight(si: &SysInfo) -> bool {
|
||||
si.ram_mib > 0
|
||||
&& si.model_bytes > 0
|
||||
&& si.model_bytes as f64 > 0.7 * si.ram_mib as f64 * 1048576.0
|
||||
}
|
||||
|
||||
/// `ckptArgs`.
|
||||
pub fn ckpt_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (Vec<String>, Vec<String>) {
|
||||
let mut args: Vec<String> = Vec::new();
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
let hybrid = gguf_hybrid(si.gguf.as_ref());
|
||||
let cp_flags = ["-ctxcp", "--ctx-checkpoints", "--swa-checkpoints"];
|
||||
|
||||
let v = trim(get(cfg, "CTX_CHECKPOINTS"));
|
||||
if !v.is_empty() {
|
||||
match atoi(v) {
|
||||
Some(n) if n >= 0 => {
|
||||
if !si.help.contains("--ctx-checkpoints") {
|
||||
notes.push(
|
||||
"ce moteur ne connaît pas --ctx-checkpoints : CTX_CHECKPOINTS ignoré"
|
||||
.into(),
|
||||
);
|
||||
} else if has_any_flag(extra, &cp_flags) {
|
||||
} else if !si.env("LLAMA_ARG_CTX_CHECKPOINTS").is_empty() {
|
||||
notes.push(
|
||||
"LLAMA_ARG_CTX_CHECKPOINTS est posée : CTX_CHECKPOINTS ignoré".into(),
|
||||
);
|
||||
} else {
|
||||
args.push("--ctx-checkpoints".into());
|
||||
args.push(n.to_string());
|
||||
}
|
||||
}
|
||||
_ => notes.push(format!(
|
||||
"CTX_CHECKPOINTS={v} illisible (un nombre de points de reprise par slot, ≥ 0) : ignoré"
|
||||
)),
|
||||
}
|
||||
} else if hybrid
|
||||
&& ram_tight(si)
|
||||
&& si.help.contains("--ctx-checkpoints")
|
||||
&& !has_any_flag(extra, &cp_flags)
|
||||
&& si.env("LLAMA_ARG_CTX_CHECKPOINTS").is_empty()
|
||||
{
|
||||
notes.push(format!(
|
||||
"modèle hybride de {} Gio pour {} Gio de RAM : jusqu'à {ENGINE_CKPT_DEFAULT} points de reprise (~{} Mio chacun) prennent sur le cache de pages des poids. CTX_CHECKPOINTS=8 à 16 si le débit baisse.",
|
||||
si.model_bytes >> 30,
|
||||
si.ram_mib >> 10,
|
||||
ckpt_mib(si.gguf.as_ref())
|
||||
));
|
||||
}
|
||||
|
||||
let step_user = has_any_flag(extra, &["-cms", "--checkpoint-min-step"])
|
||||
|| !si.env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").is_empty();
|
||||
let v = trim(get(cfg, "CKPT_MIN_STEP"));
|
||||
if !v.is_empty() {
|
||||
match atoi(v) {
|
||||
Some(n) if n > 0 => {
|
||||
if !si.help.contains("--checkpoint-min-step") {
|
||||
notes.push(
|
||||
"ce moteur ne connaît pas --checkpoint-min-step : CKPT_MIN_STEP ignoré"
|
||||
.into(),
|
||||
);
|
||||
} else if has_any_flag(extra, &["-cms", "--checkpoint-min-step"]) {
|
||||
} else if !si.env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").is_empty() {
|
||||
notes.push(
|
||||
"LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT est posée : CKPT_MIN_STEP ignoré"
|
||||
.into(),
|
||||
);
|
||||
} else {
|
||||
args.push("--checkpoint-min-step".into());
|
||||
args.push(n.to_string());
|
||||
}
|
||||
}
|
||||
_ => notes.push(format!(
|
||||
"CKPT_MIN_STEP={v} refusé (un nombre de jetons > 0 ; 0 laisserait le moteur jeter le point de reprise du début de la conversation) : défaut du moteur"
|
||||
)),
|
||||
}
|
||||
return (args, notes);
|
||||
}
|
||||
|
||||
let notice = engine_build_notice(si.engine_build);
|
||||
if !hybrid || notice.is_empty() {
|
||||
return (args, notes);
|
||||
}
|
||||
let n = ckpt_count(cfg, extra, si);
|
||||
if step_user || !si.help.contains("--checkpoint-min-step") || n <= 0 {
|
||||
notes.push(notice);
|
||||
return (args, notes);
|
||||
}
|
||||
if si.ram_mib <= 0 || si.model_bytes <= 0 || ram_tight(si) {
|
||||
notes.push(notice + " Pas d'espacement resserré d'office : RAM inconnue ou déjà prise par le modèle (CKPT_MIN_STEP=2048 pour l'imposer).");
|
||||
return (args, notes);
|
||||
}
|
||||
let total = n * ckpt_slots(cfg, extra);
|
||||
let worst = total * ckpt_mib(si.gguf.as_ref());
|
||||
if si.ram_avail_mib > 0 && worst > si.ram_avail_mib / 4 {
|
||||
notes.push(format!(
|
||||
"{notice} Pas d'espacement resserré d'office : {total} points de reprise (~{worst} Mio) dépasseraient le quart de la RAM libre (CKPT_MIN_STEP=2048 pour l'imposer)."
|
||||
));
|
||||
return (args, notes);
|
||||
}
|
||||
args.push("--checkpoint-min-step".into());
|
||||
args.push(CKPT_AUTO_MIN_STEP.to_string());
|
||||
notes.push(format!(
|
||||
"{notice} → --checkpoint-min-step {CKPT_AUTO_MIN_STEP} : points de reprise exacts plus rapprochés, ~{worst} Mio de RAM hôte au plus. CKPT_MIN_STEP=<jetons> pour fixer."
|
||||
));
|
||||
(args, notes)
|
||||
}
|
||||
|
||||
/// `reasoningPreserveArgs`.
|
||||
pub fn reasoning_preserve_args(
|
||||
cfg: &Cfg,
|
||||
extra: &[String],
|
||||
si: &SysInfo,
|
||||
) -> (Vec<String>, Vec<String>) {
|
||||
let v = trim(get(cfg, "REASONING_PRESERVE")).to_lowercase();
|
||||
if v.is_empty() {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
let flag = match v.as_str() {
|
||||
"on" => "--reasoning-preserve",
|
||||
"off" => "--no-reasoning-preserve",
|
||||
_ => {
|
||||
return (
|
||||
Vec::new(),
|
||||
vec![format!(
|
||||
"REASONING_PRESERVE={} illisible (on ou off) : ignoré",
|
||||
get(cfg, "REASONING_PRESERVE")
|
||||
)],
|
||||
);
|
||||
}
|
||||
};
|
||||
let one = |n: &str| (Vec::new(), vec![n.to_string()]);
|
||||
if !si.help.contains("--no-reasoning-preserve") {
|
||||
return one("ce moteur ne connaît pas --reasoning-preserve : REASONING_PRESERVE ignoré");
|
||||
}
|
||||
if has_any_flag(extra, &["--reasoning-preserve", "--no-reasoning-preserve"]) {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
if flag_values(extra, &["--chat-template-kwargs"])
|
||||
.join(" ")
|
||||
.contains("preserve_reasoning")
|
||||
{
|
||||
return one(
|
||||
"--chat-template-kwargs fixe déjà preserve_reasoning : REASONING_PRESERVE ignoré",
|
||||
);
|
||||
}
|
||||
if !si.env("LLAMA_ARG_REASONING_PRESERVE").is_empty() {
|
||||
return one("LLAMA_ARG_REASONING_PRESERVE est posée : REASONING_PRESERVE ignoré");
|
||||
}
|
||||
if si
|
||||
.env("LLAMA_ARG_CHAT_TEMPLATE_KWARGS")
|
||||
.contains("preserve_reasoning")
|
||||
{
|
||||
return one(
|
||||
"LLAMA_ARG_CHAT_TEMPLATE_KWARGS fixe déjà preserve_reasoning : REASONING_PRESERVE ignoré",
|
||||
);
|
||||
}
|
||||
(vec![flag.to_string()], Vec::new())
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Réglages d'expert du moteur local (`backend_serve_expert.go`) : marge de
|
||||
//! --fit par carte, seuil d'envoi au GPU des poids restés en RAM, branches
|
||||
//! Q/K/V en parallèle. Tous éteints tant que le preset ne les demande pas.
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, ctx_fixed, flag_value, flag_values, get, has_any_flag, help_fits_layers_itself,
|
||||
ngl_args, tensor_override, trim,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// `expertEnvKeys`.
|
||||
pub const EXPERT_ENV_KEYS: &[&str] = &["GGML_CUDA_GRAPH_OPT", "GGML_OP_OFFLOAD_MIN_BATCH"];
|
||||
/// `fitArgEnv`.
|
||||
pub const FIT_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_FIT",
|
||||
"LLAMA_ARG_FIT_TARGET",
|
||||
"LLAMA_ARG_TENSOR_SPLIT",
|
||||
];
|
||||
|
||||
pub const FIT_TARGET_MIN: i64 = 1024;
|
||||
const FIT_TARGET_MAX_VALUES: usize = 8;
|
||||
|
||||
/// `parseFitTarget` : (valeurs, remontées sous 1024, valide).
|
||||
pub fn parse_fit_target(v: &str) -> (Vec<i64>, bool, bool) {
|
||||
let parts: Vec<&str> = v.split([',', '/']).filter(|p| !p.is_empty()).collect();
|
||||
let seps = v.matches(',').count() + v.matches('/').count();
|
||||
if parts.is_empty() || parts.len() > FIT_TARGET_MAX_VALUES || seps != parts.len() - 1 {
|
||||
return (Vec::new(), false, false);
|
||||
}
|
||||
let mut vals = Vec::new();
|
||||
let mut raised = false;
|
||||
for p in parts {
|
||||
let p = trim(p);
|
||||
if p.is_empty() || !p.bytes().all(|b| b.is_ascii_digit()) || p.len() > 7 {
|
||||
return (Vec::new(), false, false);
|
||||
}
|
||||
let mut n: i64 = p.parse().unwrap_or(0);
|
||||
if n < FIT_TARGET_MIN {
|
||||
n = FIT_TARGET_MIN;
|
||||
raised = true;
|
||||
}
|
||||
vals.push(n);
|
||||
}
|
||||
(vals, raised, true)
|
||||
}
|
||||
|
||||
/// `fitTargetArgs`.
|
||||
pub fn fit_target_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (Vec<String>, Vec<String>) {
|
||||
let raw = trim(get(cfg, "FIT_TARGET"));
|
||||
if raw.is_empty() {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
let ignore = |why: &str| (Vec::new(), vec![format!("FIT_TARGET={raw} ignoré : {why}")]);
|
||||
let (vals, raised, ok) = parse_fit_target(raw);
|
||||
if !ok {
|
||||
return ignore("des Mio par carte, ex. 1024,3072 (8 valeurs au plus)");
|
||||
}
|
||||
if !si.help.contains("--fit-target") {
|
||||
return ignore("ce moteur ne connaît pas --fit-target");
|
||||
}
|
||||
if has_any_flag(extra, &["-fitt", "--fit-target"]) {
|
||||
return ignore("--fit-target déjà dans EXTRA_ARGS");
|
||||
}
|
||||
if !si.env("LLAMA_ARG_FIT_TARGET").is_empty() {
|
||||
return ignore("LLAMA_ARG_FIT_TARGET déjà posé dans l'environnement");
|
||||
}
|
||||
let (ctx, ok) = ctx_fixed(cfg, extra);
|
||||
if !ok {
|
||||
return ignore(&format!(
|
||||
"contexte « {ctx} » non fixé — --fit pourrait le réduire pour tenir ; donne un CTX chiffré"
|
||||
));
|
||||
}
|
||||
let why = fit_blocker(cfg, extra, si);
|
||||
if !why.is_empty() {
|
||||
return ignore(&format!(
|
||||
"--fit inactif ({why}), la marge ne servirait à rien"
|
||||
));
|
||||
}
|
||||
let v = vals
|
||||
.iter()
|
||||
.map(i64::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let mut note = format!(
|
||||
"--fit-target {v} (FIT_TARGET, Mio libres par carte, dans l'ordre vu par le moteur) — vérifie « offloaded N/N layers » dans le journal"
|
||||
);
|
||||
if raised {
|
||||
note.push_str(" ; marges sous 1024 Mio remontées à 1024");
|
||||
}
|
||||
(vec!["--fit-target".into(), v], vec![note])
|
||||
}
|
||||
|
||||
/// `fitBlocker` : pourquoi llama.cpp ne lancera PAS --fit (vide = il le
|
||||
/// lancera).
|
||||
pub fn fit_blocker(cfg: &Cfg, extra: &[String], si: &SysInfo) -> String {
|
||||
let mut fit = flag_value(extra, &["-fit", "--fit"]);
|
||||
if fit.is_empty() {
|
||||
fit = si.env("LLAMA_ARG_FIT").into();
|
||||
}
|
||||
if matches!(
|
||||
trim(&fit).to_lowercase().as_str(),
|
||||
"off" | "disabled" | "false" | "0"
|
||||
) {
|
||||
return "--fit off".into();
|
||||
}
|
||||
let mut ngl = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if ngl.is_empty() {
|
||||
let (a, _) = ngl_args(get(cfg, "NGL"), help_fits_layers_itself(&si.help));
|
||||
if a.len() == 2 {
|
||||
ngl = a[1].clone();
|
||||
} else {
|
||||
ngl = si.env("LLAMA_ARG_N_GPU_LAYERS").into();
|
||||
}
|
||||
}
|
||||
let ngl = trim(&ngl);
|
||||
if !ngl.is_empty() && ngl != "-1" && !ngl.eq_ignore_ascii_case("auto") {
|
||||
return format!("couches GPU fixées à {ngl}");
|
||||
}
|
||||
if has_any_flag(extra, &["-ts", "--tensor-split"])
|
||||
|| !si.env("LLAMA_ARG_TENSOR_SPLIT").is_empty()
|
||||
{
|
||||
return "--tensor-split".into();
|
||||
}
|
||||
let r = tensor_override(extra, &si.arg_env);
|
||||
if !r.is_empty() {
|
||||
return r;
|
||||
}
|
||||
let mut sm = flag_value(extra, &["-sm", "--split-mode"]);
|
||||
if sm.is_empty() {
|
||||
sm = si.env("LLAMA_ARG_SPLIT_MODE").into();
|
||||
}
|
||||
let sm = trim(&sm).to_lowercase();
|
||||
if sm == "row" || sm == "tensor" {
|
||||
return format!("-sm {sm}");
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `opOffloadMinBatchEnv`.
|
||||
pub fn op_offload_min_batch_env(cfg: &Cfg, si: &SysInfo) -> (String, Vec<String>) {
|
||||
let raw = trim(get(cfg, "OP_OFFLOAD_MIN_BATCH"));
|
||||
if raw.is_empty() || !si.uenv("GGML_OP_OFFLOAD_MIN_BATCH").is_empty() {
|
||||
return (String::new(), Vec::new());
|
||||
}
|
||||
match super::atoi(raw) {
|
||||
Some(n) if (1..=1 << 20).contains(&n) => {
|
||||
let v = n.to_string();
|
||||
(
|
||||
v.clone(),
|
||||
vec![format!(
|
||||
"GGML_OP_OFFLOAD_MIN_BATCH={v} (OP_OFFLOAD_MIN_BATCH) : lots de moins de {v} jetons calculés sur CPU pour les poids restés en RAM"
|
||||
)],
|
||||
)
|
||||
}
|
||||
_ => (
|
||||
String::new(),
|
||||
vec![format!(
|
||||
"OP_OFFLOAD_MIN_BATCH={raw} ignoré : un nombre de jetons, ex. 128"
|
||||
)],
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// `graphOptEnv`.
|
||||
pub fn graph_opt_env(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (String, Vec<String>) {
|
||||
let raw = trim(get(cfg, "CUDA_GRAPH_OPT"));
|
||||
match raw.to_lowercase().as_str() {
|
||||
"" | "off" => return (String::new(), Vec::new()),
|
||||
"on" => {}
|
||||
_ => {
|
||||
return (
|
||||
String::new(),
|
||||
vec![format!("CUDA_GRAPH_OPT={raw} ignoré (on ou off)")],
|
||||
);
|
||||
}
|
||||
}
|
||||
if !si.uenv("GGML_CUDA_GRAPH_OPT").is_empty() {
|
||||
return (String::new(), Vec::new());
|
||||
}
|
||||
let why = qkv_split_risk(extra, &si.arg_env);
|
||||
if !why.is_empty() {
|
||||
return (
|
||||
String::new(),
|
||||
vec![format!(
|
||||
"CUDA_GRAPH_OPT ignoré : {why} peut séparer Q, K et V d'une couche entre cartes"
|
||||
)],
|
||||
);
|
||||
}
|
||||
("1".into(), vec!["GGML_CUDA_GRAPH_OPT=1 (CUDA_GRAPH_OPT, expérimental) — si le moteur s'arrête sur un GGML_ASSERT, CUDA_GRAPH_OPT=off puis redémarre".into()])
|
||||
}
|
||||
|
||||
/// `qkvSplitRisk`.
|
||||
pub fn qkv_split_risk(extra: &[String], arg_env: &HashMap<String, String>) -> String {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
let mut sm = flag_value(extra, &["-sm", "--split-mode"]);
|
||||
if sm.is_empty() {
|
||||
sm = env("LLAMA_ARG_SPLIT_MODE").into();
|
||||
}
|
||||
let sm = trim(&sm).to_lowercase();
|
||||
if sm == "row" || sm == "tensor" {
|
||||
return format!("-sm {sm}");
|
||||
}
|
||||
let mut vals = flag_values(extra, &["-ot", "--override-tensor"]);
|
||||
vals.push(env("LLAMA_ARG_OVERRIDE_TENSOR").to_string());
|
||||
if vals.iter().any(|v| v.to_lowercase().contains("attn")) {
|
||||
return "-ot sur l'attention".into();
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! Garde-fous de fidélité (`backend_serve_fidelity.go`) : Loki ne choisit
|
||||
//! jamais un réglage qui change ce que le modèle calcule, mais il DIT ceux
|
||||
//! que la ligne finale porte (cache KV quantifié, glissement de contexte,
|
||||
//! réutilisation approchée du cache, acceptation synthétique).
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, env_truthy, flag_value, has_any_flag, has_flag_prefix, serve_kv_types,
|
||||
tensor_override, trim,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// `fidelityArgEnv`.
|
||||
pub const FIDELITY_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_CACHE_TYPE_K",
|
||||
"LLAMA_ARG_CACHE_TYPE_V",
|
||||
"LLAMA_ARG_CONTEXT_SHIFT",
|
||||
"LLAMA_ARG_NO_CONTEXT_SHIFT",
|
||||
"LLAMA_ARG_CACHE_REUSE",
|
||||
"LLAMA_ARG_MMPROJ",
|
||||
"LLAMA_ARG_MMPROJ_URL",
|
||||
"LLAMA_ARG_SPEC_SYNTH_LEN",
|
||||
];
|
||||
|
||||
/// `effectiveKVTypes` : (K, V, source la plus forte).
|
||||
pub fn effective_kv_types(
|
||||
cfg: &Cfg,
|
||||
extra: &[String],
|
||||
arg_env: &HashMap<String, String>,
|
||||
) -> (String, String, String) {
|
||||
let env = |k: &str| arg_env.get(k).cloned().unwrap_or_default();
|
||||
let (mut k, mut v) = (env("LLAMA_ARG_CACHE_TYPE_K"), env("LLAMA_ARG_CACHE_TYPE_V"));
|
||||
let mut src = String::new();
|
||||
if !k.is_empty() || !v.is_empty() {
|
||||
src = "défini par LLAMA_ARG_CACHE_TYPE_K/V".into();
|
||||
}
|
||||
let (ck, cv) = serve_kv_types(cfg);
|
||||
if !ck.is_empty() || !cv.is_empty() {
|
||||
src = "KV_TYPE".into();
|
||||
if !ck.is_empty() {
|
||||
k = ck;
|
||||
}
|
||||
if !cv.is_empty() {
|
||||
v = cv;
|
||||
}
|
||||
}
|
||||
let sk = flag_value(extra, &["-ctk", "--cache-type-k"]);
|
||||
if !sk.is_empty() {
|
||||
k = sk;
|
||||
src = "défini par EXTRA_ARGS".into();
|
||||
}
|
||||
let sv = flag_value(extra, &["-ctv", "--cache-type-v"]);
|
||||
if !sv.is_empty() {
|
||||
v = sv;
|
||||
src = "défini par EXTRA_ARGS".into();
|
||||
}
|
||||
(k, v, src)
|
||||
}
|
||||
|
||||
/// `kvFidelity` : rang (0 = mêmes calculs que f16) et libellé.
|
||||
pub fn kv_fidelity(t: &str) -> (i64, &'static str) {
|
||||
match trim(t).to_lowercase().as_str() {
|
||||
"" | "f16" | "f32" => (0, ""),
|
||||
"bf16" => (1, "numérique différente de f16 (mantisse plus courte)"),
|
||||
"q8_0" | "q5_0" | "q5_1" => (2, "modifie légèrement les sorties par rapport à f16"),
|
||||
"q4_0" | "q4_1" | "iq4_nl" => (3, "perte mesurable par rapport à f16"),
|
||||
_ => (2, "modifie les sorties par rapport à f16"),
|
||||
}
|
||||
}
|
||||
|
||||
fn or_f16(t: &str) -> &str {
|
||||
if t.is_empty() { "f16" } else { t }
|
||||
}
|
||||
|
||||
/// `kvFidelityNote`.
|
||||
pub fn kv_fidelity_note(
|
||||
k: &str,
|
||||
v: &str,
|
||||
src: &str,
|
||||
extra: &[String],
|
||||
arg_env: &HashMap<String, String>,
|
||||
) -> String {
|
||||
let (rk, lk) = kv_fidelity(k);
|
||||
let (rv, lv) = kv_fidelity(v);
|
||||
if rk == 0 && rv == 0 {
|
||||
return String::new();
|
||||
}
|
||||
let label = if rv > rk { lv } else { lk };
|
||||
let mut note = format!(
|
||||
"cache KV {}/{} ({src}) : {label} — choix de l'utilisateur, laissé tel quel.",
|
||||
or_f16(k),
|
||||
or_f16(v)
|
||||
);
|
||||
if !tensor_override(extra, arg_env).is_empty() {
|
||||
note.push_str(" Repasser en f16 demande plus de VRAM, et le placement fixé par -ot/--n-cpu-moe empêche --fit de compenser : il faudrait sans doute relever --n-cpu-moe.");
|
||||
}
|
||||
note
|
||||
}
|
||||
|
||||
/// `lossyCacheNotes`.
|
||||
pub fn lossy_cache_notes(extra: &[String], si: &SysInfo) -> Vec<String> {
|
||||
let mut notes = Vec::new();
|
||||
let ignored = if !si.mmproj.is_empty()
|
||||
|| has_any_flag(extra, &["-mm", "--mmproj", "-mmu", "--mmproj-url"])
|
||||
|| !si.env("LLAMA_ARG_MMPROJ").is_empty()
|
||||
|| !si.env("LLAMA_ARG_MMPROJ_URL").is_empty()
|
||||
{
|
||||
" (vision chargée : llama.cpp l'ignore de toute façon)"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let (on, src) = context_shift(extra, &si.arg_env, &si.help);
|
||||
if on && src.is_empty() {
|
||||
notes.push(format!("avertissement : ce llama-server ancien glisse le contexte par défaut — contexte plein, il jette des jetons anciens et le modèle perd une partie de la conversation sans le savoir{ignored}. --no-context-shift dans EXTRA_ARGS l'en empêche."));
|
||||
} else if on {
|
||||
notes.push(format!("avertissement : --context-shift ({src}) — contexte plein, le moteur jette des jetons anciens et le modèle perd une partie de la conversation sans le savoir{ignored}."));
|
||||
}
|
||||
let (mut sv, mut src) = (flag_value(extra, &["--cache-reuse"]), "EXTRA_ARGS");
|
||||
if sv.is_empty() {
|
||||
sv = si.env("LLAMA_ARG_CACHE_REUSE").into();
|
||||
src = "LLAMA_ARG_CACHE_REUSE";
|
||||
}
|
||||
if let Some(n) = super::atoi(trim(&sv))
|
||||
&& n > 0
|
||||
{
|
||||
notes.push(format!("avertissement : --cache-reuse {n} ({src}) — réutilise des morceaux de cache calculés sous un autre préfixe : les sorties ne sont plus exactement celles d'un calcul complet{ignored}."));
|
||||
}
|
||||
let mut f = has_flag_prefix(extra, &["--spec-synth-"]);
|
||||
if !f.is_empty() || !si.env("LLAMA_ARG_SPEC_SYNTH_LEN").is_empty() {
|
||||
if f.is_empty() {
|
||||
f = "LLAMA_ARG_SPEC_SYNTH_LEN".into();
|
||||
}
|
||||
notes.push(format!("avertissement : {f} — le moteur accepte des jetons du brouillon AU HASARD, sans vérification : les réponses ne sont plus celles du modèle. Réservé aux mesures, retire-le."));
|
||||
}
|
||||
notes
|
||||
}
|
||||
|
||||
/// `contextShift` : (glisse ?, qui l'a décidé — vide : son défaut).
|
||||
pub fn context_shift(
|
||||
extra: &[String],
|
||||
arg_env: &HashMap<String, String>,
|
||||
help: &str,
|
||||
) -> (bool, String) {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
let (mut on, mut set) = (false, false);
|
||||
for a in extra {
|
||||
let name = a.split_once('=').map_or(a.as_str(), |(n, _)| n);
|
||||
match name {
|
||||
"--context-shift" => {
|
||||
on = true;
|
||||
set = true;
|
||||
}
|
||||
"--no-context-shift" => {
|
||||
on = false;
|
||||
set = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if set {
|
||||
return (on, "EXTRA_ARGS".into());
|
||||
}
|
||||
if help.contains("--no-context-shift") && !help.contains("--context-shift") {
|
||||
if env_truthy(env("LLAMA_ARG_NO_CONTEXT_SHIFT")) {
|
||||
return (false, "LLAMA_ARG_NO_CONTEXT_SHIFT".into());
|
||||
}
|
||||
return (true, String::new());
|
||||
}
|
||||
if env_truthy(env("LLAMA_ARG_CONTEXT_SHIFT")) {
|
||||
return (true, "LLAMA_ARG_CONTEXT_SHIFT".into());
|
||||
}
|
||||
(false, String::new())
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
//! `loki serve` — port de `backend_serve*.go` (lot 14 de la migration) :
|
||||
//! composer la ligne de commande de llama-server à partir du preset, puis
|
||||
//! céder la place au moteur (`exec`).
|
||||
//!
|
||||
//! Tout ce qui décide est PUR et vit ici ([`build_serve_args`] et ses
|
||||
//! modules) : une table de lancements tirée du Go
|
||||
//! (`internal/loki/parity_serve_test.go`) le vérifie argument par argument.
|
||||
//! [`run`] fait les lectures du monde (aide du moteur, GGUF, RAM, VRAM,
|
||||
//! cgroup, variables) avant d'appeler la partie pure, comme `cmdServe`.
|
||||
//!
|
||||
//! L'ordre des arguments fait partie du comportement : llama-server retient
|
||||
//! la DERNIÈRE occurrence d'un drapeau.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use crate::gguf::GgufMeta;
|
||||
|
||||
pub mod cache;
|
||||
pub mod ckpt;
|
||||
pub mod expert;
|
||||
pub mod fidelity;
|
||||
pub mod moe;
|
||||
pub mod persist;
|
||||
pub mod queues;
|
||||
pub mod run;
|
||||
pub mod side;
|
||||
pub mod spec;
|
||||
pub mod split;
|
||||
pub mod threads;
|
||||
|
||||
pub use cache::*;
|
||||
pub use ckpt::*;
|
||||
pub use expert::*;
|
||||
pub use fidelity::*;
|
||||
pub use moe::*;
|
||||
pub use persist::*;
|
||||
pub use queues::*;
|
||||
pub use side::*;
|
||||
pub use spec::*;
|
||||
pub use split::*;
|
||||
pub use threads::*;
|
||||
|
||||
/// Configuration d'un preset (`ReadConfig()`).
|
||||
pub type Cfg = HashMap<String, String>;
|
||||
|
||||
/// Ce que `cmdServe` a dû demander au monde avant de composer la ligne de
|
||||
/// commande (`serveSysInfo`). Zéro / vide = inconnu.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SysInfo {
|
||||
/// Sortie de `<bin> --help` ; vide = moteur inconnu, aucun risque pris.
|
||||
pub help: String,
|
||||
/// Chemin du .gguf principal, résolu et vérifié.
|
||||
pub model: String,
|
||||
/// Projecteur vision résolu ; vide = pas de vision.
|
||||
pub mmproj: String,
|
||||
/// Clé d'API effective ; vide = serveur ouvert.
|
||||
pub api_key: String,
|
||||
/// Sonde de conteneur (Linux).
|
||||
pub cpu: CpuBudget,
|
||||
/// `CUDA_SCALE_LAUNCH_QUEUES` déjà posée : intouchée.
|
||||
pub launch_queues: String,
|
||||
/// GPU visibles ; 0 = inconnu.
|
||||
pub gpus: i64,
|
||||
/// `LLAMA_ARG_*` de l'environnement qui décident de quelque chose.
|
||||
pub arg_env: HashMap<String, String>,
|
||||
/// `GGML_*` déjà posées par l'utilisateur.
|
||||
pub user_env: HashMap<String, String>,
|
||||
pub gguf: Option<GgufMeta>,
|
||||
pub model_bytes: i64,
|
||||
pub ram_mib: i64,
|
||||
pub ram_avail_mib: i64,
|
||||
pub vram_mib: i64,
|
||||
pub unified_mem: bool,
|
||||
pub engine_vram_mib: i64,
|
||||
pub slot_dir: String,
|
||||
pub engine_build: i64,
|
||||
pub draft: String,
|
||||
pub draft_err: String,
|
||||
pub draft_gguf: Option<GgufMeta>,
|
||||
pub draft_is_model: bool,
|
||||
pub spec_auto_blocked: String,
|
||||
pub split_devs: Vec<SplitDev>,
|
||||
}
|
||||
|
||||
impl SysInfo {
|
||||
pub(crate) fn env(&self, k: &str) -> &str {
|
||||
self.arg_env.get(k).map_or("", String::as_str)
|
||||
}
|
||||
pub(crate) fn uenv(&self, k: &str) -> &str {
|
||||
self.user_env.get(k).map_or("", String::as_str)
|
||||
}
|
||||
pub(crate) fn blocks(&self) -> i64 {
|
||||
self.gguf.as_ref().map_or(0, |g| g.block_count)
|
||||
}
|
||||
}
|
||||
|
||||
/// Une carte telle que la voit le moteur (`--list-devices`), dans son ordre.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct SplitDev {
|
||||
pub id: String,
|
||||
pub total_mib: i64,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Petits outils au comportement du Go
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `strings.TrimSpace`.
|
||||
pub(crate) fn trim(s: &str) -> &str {
|
||||
s.trim_matches(char::is_whitespace)
|
||||
}
|
||||
|
||||
/// `strconv.Atoi`.
|
||||
pub(crate) fn atoi(s: &str) -> Option<i64> {
|
||||
s.parse::<i64>().ok()
|
||||
}
|
||||
|
||||
/// `cfg[k]` (vide si absent).
|
||||
pub(crate) fn get<'a>(cfg: &'a Cfg, k: &str) -> &'a str {
|
||||
cfg.get(k).map_or("", String::as_str)
|
||||
}
|
||||
|
||||
/// `hasAnyFlag` : l'un de ces drapeaux est-il écrit (forme `--x=v` comprise) ?
|
||||
pub fn has_any_flag(args: &[String], flags: &[&str]) -> bool {
|
||||
args.iter().any(|a| {
|
||||
let name = match a.find('=') {
|
||||
Some(i) if i > 0 => &a[..i],
|
||||
_ => a.as_str(),
|
||||
};
|
||||
flags.contains(&name)
|
||||
})
|
||||
}
|
||||
|
||||
/// `flagValue` : valeur de la DERNIÈRE occurrence (`-t 6` ou `-t=6`).
|
||||
pub fn flag_value(args: &[String], flags: &[&str]) -> String {
|
||||
let mut v = String::new();
|
||||
for (i, a) in args.iter().enumerate() {
|
||||
let (name, val) = match a.split_once('=') {
|
||||
Some((n, v)) => (n, Some(v)),
|
||||
None => (a.as_str(), None),
|
||||
};
|
||||
if !flags.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
match val {
|
||||
Some(x) => v = x.to_string(),
|
||||
None => {
|
||||
if let Some(n) = args.get(i + 1) {
|
||||
v = n.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// `flagValues` : les valeurs de TOUTES les occurrences.
|
||||
pub fn flag_values(args: &[String], flags: &[&str]) -> Vec<String> {
|
||||
let mut vals = Vec::new();
|
||||
for (i, a) in args.iter().enumerate() {
|
||||
let (name, val) = match a.split_once('=') {
|
||||
Some((n, v)) => (n, Some(v)),
|
||||
None => (a.as_str(), None),
|
||||
};
|
||||
if !flags.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
match val {
|
||||
Some(x) => vals.push(x.to_string()),
|
||||
None => {
|
||||
if let Some(n) = args.get(i + 1) {
|
||||
vals.push(n.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
vals
|
||||
}
|
||||
|
||||
/// `hasFlagPrefix` : le nom du premier drapeau qui commence par l'un des
|
||||
/// préfixes, ou vide.
|
||||
pub fn has_flag_prefix(args: &[String], prefixes: &[&str]) -> String {
|
||||
for a in args {
|
||||
for p in prefixes {
|
||||
if a.starts_with(p) {
|
||||
return a.split_once('=').map_or(a.as_str(), |(n, _)| n).to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `envTruthy` : on, enabled, true, 1.
|
||||
pub fn env_truthy(v: &str) -> bool {
|
||||
matches!(trim(v), "on" | "enabled" | "true" | "1")
|
||||
}
|
||||
|
||||
/// `argValue` : valeur de la dernière occurrence de `flag` (forme séparée).
|
||||
pub fn arg_value(args: &[String], flag: &str) -> String {
|
||||
let mut v = String::new();
|
||||
for i in 0..args.len().saturating_sub(1) {
|
||||
if args[i] == flag {
|
||||
v = args[i + 1].clone();
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// `firstNonEmpty`.
|
||||
pub(crate) fn first_non_empty<'a>(vals: &[&'a str]) -> &'a str {
|
||||
vals.iter()
|
||||
.copied()
|
||||
.find(|v| !trim(v).is_empty())
|
||||
.unwrap_or("")
|
||||
}
|
||||
|
||||
/// `isExternalConfig`.
|
||||
pub fn is_external(cfg: &Cfg) -> bool {
|
||||
trim(get(cfg, "EXTERNAL")) == "1"
|
||||
}
|
||||
|
||||
/// `isStrataConfig` (`engineOf`).
|
||||
pub fn is_strata(cfg: &Cfg) -> bool {
|
||||
matches!(
|
||||
trim(get(cfg, "ENGINE")).to_lowercase().as_str(),
|
||||
"strata" | "moe"
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn s(v: &[&str]) -> Vec<String> {
|
||||
v.iter().map(|x| x.to_string()).collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Aide du moteur
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `helpSupportsReasoningFlag`.
|
||||
pub fn help_supports_reasoning_flag(help: &str) -> bool {
|
||||
help.contains("--reasoning ")
|
||||
}
|
||||
|
||||
/// `helpSupportsLoadMode`.
|
||||
pub fn help_supports_load_mode(help: &str) -> bool {
|
||||
help.contains("--load-mode")
|
||||
}
|
||||
|
||||
/// `helpSupportsNGLAuto` : « 'auto' » dans les 400 octets qui suivent le
|
||||
/// drapeau des couches GPU.
|
||||
pub fn help_supports_ngl_auto(h: &str) -> bool {
|
||||
let i = match h.find("--n-gpu-layers").or_else(|| h.find("-ngl")) {
|
||||
Some(i) => i,
|
||||
None => return false,
|
||||
};
|
||||
let end = (i + 400).min(h.len());
|
||||
// Tranche d'octets (le Go découpe sans égard aux runes).
|
||||
String::from_utf8_lossy(&h.as_bytes()[i..end]).contains("'auto'")
|
||||
}
|
||||
|
||||
/// `helpFitsLayersItself`.
|
||||
pub fn help_fits_layers_itself(help: &str) -> bool {
|
||||
help_supports_ngl_auto(help) || help_supports_load_mode(help)
|
||||
}
|
||||
|
||||
const NGL_999_NOTE: &str = "NGL=999 (sentinelle « toutes les couches ») → -ngl auto : ce moteur mesure lui-même la VRAM libre. NGL=all pour forcer l'ancien comportement.";
|
||||
|
||||
/// `nglArgs`.
|
||||
pub fn ngl_args(ngl: &str, fits_itself: bool) -> (Vec<String>, String) {
|
||||
let ngl = trim(ngl);
|
||||
if ngl.eq_ignore_ascii_case("auto") {
|
||||
(Vec::new(), String::new())
|
||||
} else if ngl == "999" && fits_itself {
|
||||
(s(&["-ngl", "auto"]), NGL_999_NOTE.into())
|
||||
} else if !ngl.is_empty() {
|
||||
(s(&["-ngl", ngl]), String::new())
|
||||
} else if fits_itself {
|
||||
(s(&["-ngl", "auto"]), String::new())
|
||||
} else {
|
||||
(s(&["-ngl", "999"]), String::new())
|
||||
}
|
||||
}
|
||||
|
||||
/// `normalizeLoadFlags`.
|
||||
pub fn normalize_load_flags(args: &[String], supports_load_mode: bool) -> (Vec<String>, String) {
|
||||
if !supports_load_mode {
|
||||
return downgrade_load_mode(args);
|
||||
}
|
||||
let explicit = has_any_flag(args, &["--load-mode", "-lm"]);
|
||||
let (mut mlock, mut nommap, mut mmap) = (false, false, false);
|
||||
let mut out = Vec::with_capacity(args.len() + 2);
|
||||
for a in args {
|
||||
match a.as_str() {
|
||||
"--mlock" => mlock = true,
|
||||
"--no-mmap" => nommap = true,
|
||||
"--mmap" => mmap = true,
|
||||
_ => out.push(a.clone()),
|
||||
}
|
||||
}
|
||||
if explicit || !(mlock || nommap || mmap) {
|
||||
return (out, String::new());
|
||||
}
|
||||
let mode = if mlock && nommap {
|
||||
"mlock"
|
||||
} else if mlock {
|
||||
"mmap+mlock"
|
||||
} else if nommap {
|
||||
"none"
|
||||
} else {
|
||||
"mmap"
|
||||
};
|
||||
out.push("--load-mode".into());
|
||||
out.push(mode.into());
|
||||
(out, String::new())
|
||||
}
|
||||
|
||||
/// `downgradeLoadMode`.
|
||||
pub fn downgrade_load_mode(args: &[String]) -> (Vec<String>, String) {
|
||||
let (mut mode, mut found) = (String::new(), false);
|
||||
let mut kept: Vec<String> = Vec::with_capacity(args.len());
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
let a = &args[i];
|
||||
if a == "--load-mode" || a == "-lm" {
|
||||
found = true;
|
||||
if i + 1 < args.len() {
|
||||
mode = args[i + 1].to_lowercase();
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(v) = a.strip_prefix("--load-mode=") {
|
||||
found = true;
|
||||
mode = v.to_lowercase();
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
kept.push(a.clone());
|
||||
i += 1;
|
||||
}
|
||||
if !found {
|
||||
return (args.to_vec(), String::new());
|
||||
}
|
||||
let add = |kept: &mut Vec<String>, flag: &str| {
|
||||
if !kept.iter().any(|a| a == flag) {
|
||||
kept.push(flag.into());
|
||||
}
|
||||
};
|
||||
match mode.as_str() {
|
||||
"none" => add(&mut kept, "--no-mmap"),
|
||||
"mlock" => {
|
||||
add(&mut kept, "--mlock");
|
||||
add(&mut kept, "--no-mmap");
|
||||
}
|
||||
"mmap+mlock" => add(&mut kept, "--mlock"),
|
||||
"dio" => {
|
||||
return (
|
||||
kept,
|
||||
"ce moteur ne connaît pas --load-mode dio (DirectIO) : chargement par défaut"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
(kept, String::new())
|
||||
}
|
||||
|
||||
/// `cudaDeviceEnv`.
|
||||
pub fn cuda_device_env(cfg: &Cfg) -> BTreeMap<String, String> {
|
||||
let mut env = BTreeMap::new();
|
||||
let v = get(cfg, "CUDA_VISIBLE_DEVICES");
|
||||
if !v.is_empty() {
|
||||
env.insert("CUDA_VISIBLE_DEVICES".into(), v.into());
|
||||
env.insert("CUDA_DEVICE_ORDER".into(), "PCI_BUS_ID".into());
|
||||
}
|
||||
env
|
||||
}
|
||||
|
||||
/// `serveKVTypes` : KV_TYPE pour les deux, KV_TYPE_K / KV_TYPE_V à part.
|
||||
pub fn serve_kv_types(cfg: &Cfg) -> (String, String) {
|
||||
let kv = get(cfg, "KV_TYPE");
|
||||
let (mut k, mut v) = (kv.to_string(), kv.to_string());
|
||||
let sk = get(cfg, "KV_TYPE_K");
|
||||
if !sk.is_empty() {
|
||||
k = sk.into();
|
||||
}
|
||||
let sv = get(cfg, "KV_TYPE_V");
|
||||
if !sv.is_empty() {
|
||||
v = sv.into();
|
||||
}
|
||||
(k, v)
|
||||
}
|
||||
|
||||
/// Ligne de commande, environnement et notes d'un lancement.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Launch {
|
||||
pub args: Vec<String>,
|
||||
pub env: BTreeMap<String, String>,
|
||||
pub notes: Vec<String>,
|
||||
}
|
||||
|
||||
/// `buildServeArgs` : la ligne de commande de llama-server (`bin` en tête),
|
||||
/// l'environnement à poser et les notes. Fonction pure.
|
||||
pub fn build_serve_args(cfg: &Cfg, extra: &[String], bin: &str, si: &SysInfo) -> Launch {
|
||||
let g = |key: &str, fallback: &str| -> String {
|
||||
match cfg.get(key) {
|
||||
Some(v) if !v.is_empty() => v.clone(),
|
||||
_ => fallback.to_string(),
|
||||
}
|
||||
};
|
||||
let (ktv, vtv) = serve_kv_types(cfg);
|
||||
let mut env = cuda_device_env(cfg);
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
|
||||
let (extra, load_note) = normalize_load_flags(extra, help_supports_load_mode(&si.help));
|
||||
let extra = &extra[..];
|
||||
if !load_note.is_empty() {
|
||||
notes.push(load_note);
|
||||
}
|
||||
|
||||
let st = split_tensor_args(cfg, extra, si);
|
||||
let mut place: Vec<String> = extra.to_vec();
|
||||
if st.on {
|
||||
place.push("-sm".into());
|
||||
place.push("tensor".into());
|
||||
}
|
||||
for (k, v) in &st.env {
|
||||
env.insert(k.clone(), v.clone());
|
||||
}
|
||||
notes.extend(st.notes.iter().cloned());
|
||||
|
||||
let (q, q_notes) = launch_queues_env(cfg, &place, si);
|
||||
if !q.is_empty() {
|
||||
env.insert("CUDA_SCALE_LAUNCH_QUEUES".into(), q);
|
||||
}
|
||||
notes.extend(q_notes);
|
||||
let (gv, g_notes) = graph_opt_env(cfg, &place, si);
|
||||
if !gv.is_empty() {
|
||||
env.insert("GGML_CUDA_GRAPH_OPT".into(), gv);
|
||||
}
|
||||
let (ov, o_notes) = op_offload_min_batch_env(cfg, si);
|
||||
if !ov.is_empty() {
|
||||
env.insert("GGML_OP_OFFLOAD_MIN_BATCH".into(), ov);
|
||||
}
|
||||
notes.extend(g_notes);
|
||||
notes.extend(o_notes);
|
||||
|
||||
let (threads, thread_notes) = thread_args(
|
||||
get(cfg, "THREADS"),
|
||||
get(cfg, "THREADS_BATCH"),
|
||||
extra,
|
||||
&si.cpu,
|
||||
);
|
||||
notes.extend(thread_notes);
|
||||
let (side_ctx, side_mib, side_why) = side_slot_plan(cfg, extra, si);
|
||||
let mut ctx_arg = g("CTX", "32768");
|
||||
if side_ctx > 0 {
|
||||
ctx_arg = (2 * side_ctx).to_string();
|
||||
}
|
||||
let mut args: Vec<String> = vec![
|
||||
bin.to_string(),
|
||||
"-m".into(),
|
||||
si.model.clone(),
|
||||
"-c".into(),
|
||||
ctx_arg,
|
||||
];
|
||||
args.extend(threads);
|
||||
args.extend([
|
||||
"-b".into(),
|
||||
g("BATCH", "2048"),
|
||||
"-ub".into(),
|
||||
g("UBATCH", "512"),
|
||||
"--host".into(),
|
||||
g("HOST", "0.0.0.0"),
|
||||
"--port".into(),
|
||||
g("PORT", "8080"),
|
||||
]);
|
||||
if side_ctx > 0 {
|
||||
args.extend(s(&["--parallel", "2", "--no-kv-unified"]));
|
||||
} else if !has_any_flag(extra, &["--parallel", "-np"]) {
|
||||
args.push("--parallel".into());
|
||||
args.push(g("PARALLEL", "1"));
|
||||
}
|
||||
let n = side_slot_note(side_ctx, side_mib, &side_why, ngl_forced(cfg, extra));
|
||||
if !n.is_empty() {
|
||||
notes.push(n);
|
||||
}
|
||||
if st.on {
|
||||
args.extend(st.ngl.iter().cloned());
|
||||
} else if !has_any_flag(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]) {
|
||||
let (ngl, note) = ngl_args(&g("NGL", ""), help_fits_layers_itself(&si.help));
|
||||
if !note.is_empty() {
|
||||
notes.push(note);
|
||||
}
|
||||
args.extend(ngl);
|
||||
}
|
||||
let (fitt, fit_notes) = fit_target_args(cfg, &place, si);
|
||||
args.extend(fitt);
|
||||
args.extend(st.args.iter().cloned());
|
||||
notes.extend(fit_notes);
|
||||
if !ktv.is_empty() {
|
||||
args.push("-ctk".into());
|
||||
args.push(ktv);
|
||||
}
|
||||
if !vtv.is_empty() {
|
||||
args.push("-ctv".into());
|
||||
args.push(vtv);
|
||||
}
|
||||
if !si.mmproj.is_empty() {
|
||||
args.push("--mmproj".into());
|
||||
args.push(si.mmproj.clone());
|
||||
}
|
||||
let r = trim(get(cfg, "REASONING"));
|
||||
if !r.is_empty() {
|
||||
if crate::chat::wire::reasoning_active(r) {
|
||||
args.extend([
|
||||
"--reasoning".into(),
|
||||
r.to_string(),
|
||||
"--reasoning-budget".into(),
|
||||
g("REASONING_BUDGET", "-1"),
|
||||
]);
|
||||
} else if help_supports_reasoning_flag(&si.help) {
|
||||
args.extend(s(&["--reasoning", "off"]));
|
||||
} else {
|
||||
notes.push(
|
||||
"ce moteur ne connaît pas --reasoning : impossible de désactiver le raisonnement"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let (rp, rp_notes) = reasoning_preserve_args(cfg, extra, si);
|
||||
args.extend(rp);
|
||||
notes.extend(rp_notes);
|
||||
if !si.api_key.is_empty() {
|
||||
args.push("--api-key".into());
|
||||
args.push(si.api_key.clone());
|
||||
}
|
||||
let (cram, cram_notes) = cache_ram_args(cfg, extra, si);
|
||||
args.extend(cram);
|
||||
args.extend(slot_save_args(cfg, extra, si));
|
||||
notes.extend(cram_notes);
|
||||
let (ok, why) = slot_persist_plan(cfg, extra, si);
|
||||
let n = slot_persist_note(ok, &why);
|
||||
if !n.is_empty() {
|
||||
notes.push(n);
|
||||
}
|
||||
let (ck, ck_notes) = ckpt_args(cfg, extra, si);
|
||||
args.extend(ck);
|
||||
notes.extend(ck_notes);
|
||||
let sp = spec_args(cfg, extra, si);
|
||||
args.extend(sp.args);
|
||||
notes.extend(sp.notes);
|
||||
args.extend(extra.iter().cloned());
|
||||
let (ek, ev, src) = effective_kv_types(cfg, extra, &si.arg_env);
|
||||
let n = kv_fidelity_note(&ek, &ev, &src, extra, &si.arg_env);
|
||||
if !n.is_empty() {
|
||||
notes.push(n);
|
||||
}
|
||||
notes.extend(lossy_cache_notes(extra, si));
|
||||
notes.extend(moe_notes(cfg, extra, si));
|
||||
let (w, _) = load_mode_risk(cfg, &args, si);
|
||||
if !w.is_empty() {
|
||||
notes.push(w);
|
||||
}
|
||||
Launch { args, env, notes }
|
||||
}
|
||||
|
||||
/// `fastKV` / `slowKV` : combinaisons accélérées en Flash-Attention CUDA.
|
||||
pub fn slow_kv(k: &str, v: &str) -> bool {
|
||||
let k = if k.is_empty() { "f16" } else { k };
|
||||
let v = if v.is_empty() { "f16" } else { v };
|
||||
!matches!(
|
||||
(k, v),
|
||||
("f16", "f16") | ("q8_0", "q8_0") | ("q4_0", "q4_0") | ("bf16", "bf16")
|
||||
)
|
||||
}
|
||||
|
||||
/// `engineCmdline` : forme copiable (variables triées en tête, guillemets
|
||||
/// autour de ce qui contient un blanc ou un guillemet, clé masquée).
|
||||
pub fn engine_cmdline(env: &BTreeMap<String, String>, args: &[String]) -> String {
|
||||
let quote = |a: &str| -> String {
|
||||
if a.is_empty() || a.contains([' ', '\t', '"', '\'']) {
|
||||
format!("\"{}\"", a.replace('"', "\\\""))
|
||||
} else {
|
||||
a.to_string()
|
||||
}
|
||||
};
|
||||
let mut out: Vec<String> = env
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}={}", quote(v)))
|
||||
.collect();
|
||||
for (i, a) in args.iter().enumerate() {
|
||||
if i > 0 && (args[i - 1] == "--api-key" || args[i - 1] == "--api-key-file") {
|
||||
out.push(quote("<clé masquée>"));
|
||||
} else {
|
||||
out.push(quote(a));
|
||||
}
|
||||
}
|
||||
out.join(" ")
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
//! Experts MoE et chargement du modèle (`backend_serve_moe.go`) : des avis et
|
||||
//! un garde-fou (LOAD_GUARD), jamais de réécriture silencieuse du preset.
|
||||
|
||||
use super::{
|
||||
Cfg, FIT_TARGET_MIN, SplitDev, SysInfo, atoi, env_truthy, first_non_empty, fit_blocker,
|
||||
flag_value, flag_values, get, has_any_flag, help_supports_load_mode, is_external, trim,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// `otPatterns`.
|
||||
pub fn ot_patterns(extra: &[String], arg_env: &HashMap<String, String>) -> Vec<String> {
|
||||
let mut vals = flag_values(extra, &["-ot", "--override-tensor"]);
|
||||
if let Some(v) = arg_env
|
||||
.get("LLAMA_ARG_OVERRIDE_TENSOR")
|
||||
.filter(|v| !v.is_empty())
|
||||
{
|
||||
vals.push(v.clone());
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for v in vals {
|
||||
for el in v.split(',') {
|
||||
let el = trim(el);
|
||||
if !el.is_empty() {
|
||||
out.push(el.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `otTargetsExperts`.
|
||||
pub fn ot_targets_experts(pattern: &str) -> bool {
|
||||
pattern.to_lowercase().contains("exps")
|
||||
}
|
||||
|
||||
/// `otOnCPU`.
|
||||
pub fn ot_on_cpu(el: &str) -> bool {
|
||||
el.split_once('=')
|
||||
.is_some_and(|(_, b)| trim(b).to_uppercase().starts_with("CPU"))
|
||||
}
|
||||
|
||||
/// `cpuExperts`.
|
||||
pub fn cpu_experts(extra: &[String], arg_env: &HashMap<String, String>) -> String {
|
||||
for el in ot_patterns(extra, arg_env) {
|
||||
let pat = el.split_once('=').map_or(el.as_str(), |(p, _)| p);
|
||||
if ot_targets_experts(pat) && ot_on_cpu(&el) {
|
||||
return "-ot experts=CPU".into();
|
||||
}
|
||||
}
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
if has_any_flag(extra, &["--cpu-moe", "-cmoe"]) || env_truthy(env("LLAMA_ARG_CPU_MOE")) {
|
||||
return "--cpu-moe".into();
|
||||
}
|
||||
let mut vals = flag_values(extra, &["-ncmoe", "--n-cpu-moe"]);
|
||||
vals.push(env("LLAMA_ARG_N_CPU_MOE").to_string());
|
||||
for v in vals {
|
||||
let v = trim(&v);
|
||||
if !v.is_empty() && v != "0" {
|
||||
return format!("--n-cpu-moe {v}");
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `helpSupportsFit` : « --fit » lui-même, pas seulement --fit-target.
|
||||
pub fn help_supports_fit(help: &str) -> bool {
|
||||
let b = help.as_bytes();
|
||||
let pat = b"--fit";
|
||||
let mut i = 0;
|
||||
while let Some(off) = help[i..].find("--fit") {
|
||||
let start = i + off;
|
||||
let end = start + pat.len();
|
||||
let before_ok =
|
||||
start == 0 || matches!(b[start - 1], b',') || b[start - 1].is_ascii_whitespace();
|
||||
let after_ok =
|
||||
end == b.len() || matches!(b[end], b',' | b'=' | b'[') || b[end].is_ascii_whitespace();
|
||||
if before_ok && after_ok {
|
||||
return true;
|
||||
}
|
||||
i = start + 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// `serveUBatch` (0 = illisible).
|
||||
pub fn serve_ubatch(cfg: &Cfg, extra: &[String]) -> i64 {
|
||||
let mut sv = flag_value(extra, &["-ub", "--ubatch-size"]);
|
||||
if sv.is_empty() {
|
||||
sv = trim(get(cfg, "UBATCH")).to_string();
|
||||
}
|
||||
if sv.is_empty() {
|
||||
sv = "512".into();
|
||||
}
|
||||
match atoi(trim(&sv)) {
|
||||
Some(n) if n >= 0 => n,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// `gibText` : des Mio en « 12,3 Gio ».
|
||||
pub fn gib_text(mib: i64) -> String {
|
||||
format!("{:.1}", mib as f64 / 1024.0).replacen('.', ",", 1) + " Gio"
|
||||
}
|
||||
|
||||
/// `moeNotes`.
|
||||
pub fn moe_notes(cfg: &Cfg, extra: &[String], si: &SysInfo) -> Vec<String> {
|
||||
let Some(g) = si.gguf.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if is_external(cfg) || g.expert_count <= 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let ub = serve_ubatch(cfg, extra);
|
||||
let r = cpu_experts(extra, &si.arg_env);
|
||||
if !r.is_empty() {
|
||||
let mut n = format!("experts placés à la main ({r}) : --fit ne les place pas");
|
||||
if ub > 0 && ub <= 512 {
|
||||
n.push_str(" ; UBATCH plus grand = tampon de calcul plus gros sur chaque GPU : monte --n-cpu-moe d'abord, ou passe au placement auto (--fit, « Dupliquer en placement auto » dans l'éditeur)");
|
||||
}
|
||||
return vec![n];
|
||||
}
|
||||
let model_mib = si.model_bytes >> 20;
|
||||
if model_mib <= 0 || si.vram_mib <= 0 || si.unified_mem || trim(&si.help).is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let cards = si.gpus.max(1);
|
||||
if model_mib <= si.vram_mib - FIT_TARGET_MIN * cards {
|
||||
return Vec::new();
|
||||
}
|
||||
let sizes = format!(
|
||||
"modèle {}, VRAM {}",
|
||||
gib_text(model_mib),
|
||||
gib_text(si.vram_mib)
|
||||
);
|
||||
if !help_supports_fit(&si.help) {
|
||||
let ngl_s = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if let Some(n) = atoi(trim(first_non_empty(&[&ngl_s, get(cfg, "NGL")])))
|
||||
&& n < 999
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
return vec![format!(
|
||||
"MoE plus gros que la VRAM ({sizes}) : ce moteur ne place pas les experts (pas de --fit) — garde --n-cpu-moe"
|
||||
)];
|
||||
}
|
||||
let why = fit_blocker(cfg, extra, si);
|
||||
if !why.is_empty() {
|
||||
if why == "surcharge de tenseurs" {
|
||||
return vec![format!(
|
||||
"MoE plus gros que la VRAM ({sizes}) et --fit coupé par un -ot qui ne vise pas les experts : rien ne les sort du GPU — --n-cpu-moe, ou retire ce -ot"
|
||||
)];
|
||||
}
|
||||
return Vec::new();
|
||||
}
|
||||
if ub > 0 && ub <= 512 {
|
||||
return vec![format!(
|
||||
"UBATCH 2048+ recommandé : --fit laisse des experts en RAM ({sizes}), recopiés vers le GPU à chaque micro-lot du prefill ; BATCH 4096 avec — à mesurer (bench complet)"
|
||||
)];
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// `loadArgEnv`.
|
||||
pub const LOAD_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_LOAD_MODE",
|
||||
"LLAMA_ARG_MLOCK",
|
||||
"LLAMA_ARG_NO_MMAP",
|
||||
"LLAMA_ARG_RPC",
|
||||
];
|
||||
|
||||
/// `residentLoadMode`.
|
||||
pub fn resident_load_mode(args: &[String], arg_env: &HashMap<String, String>) -> String {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
let mut mode = trim(&flag_value(args, &["--load-mode", "-lm"])).to_lowercase();
|
||||
let (mut mlock, mut nommap) = (
|
||||
has_any_flag(args, &["--mlock"]),
|
||||
has_any_flag(args, &["--no-mmap"]),
|
||||
);
|
||||
if mode.is_empty() && !mlock && !nommap {
|
||||
mode = trim(env("LLAMA_ARG_LOAD_MODE")).to_lowercase();
|
||||
if mode.is_empty() {
|
||||
mlock = env_truthy(env("LLAMA_ARG_MLOCK"));
|
||||
nommap = env_truthy(env("LLAMA_ARG_NO_MMAP"));
|
||||
}
|
||||
}
|
||||
if matches!(mode.as_str(), "none" | "mlock" | "dio" | "mmap+mlock") {
|
||||
return format!("--load-mode {mode}");
|
||||
}
|
||||
match (mlock, nommap) {
|
||||
(true, true) => "--mlock --no-mmap".into(),
|
||||
(true, false) => "--mlock".into(),
|
||||
(false, true) => "--no-mmap".into(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `loadGuardMarker`.
|
||||
pub const LOAD_GUARD_MARKER: &str = "LOAD_GUARD=off pour lancer quand même";
|
||||
/// `loadGuardExitStatus` (EX_CONFIG).
|
||||
pub const LOAD_GUARD_EXIT_STATUS: i32 = 78;
|
||||
|
||||
/// `loadGuardOff`.
|
||||
pub fn load_guard_off(cfg: &Cfg) -> bool {
|
||||
matches!(
|
||||
trim(get(cfg, "LOAD_GUARD")).to_lowercase().as_str(),
|
||||
"off" | "0" | "false" | "non" | "no"
|
||||
)
|
||||
}
|
||||
|
||||
/// `loadModeRisk` : (avertissement, refus).
|
||||
pub fn load_mode_risk(cfg: &Cfg, args: &[String], si: &SysInfo) -> (String, String) {
|
||||
if is_external(cfg) {
|
||||
return (String::new(), String::new());
|
||||
}
|
||||
let restricted;
|
||||
let env = if help_supports_load_mode(&si.help)
|
||||
&& (!si.env("LLAMA_ARG_MLOCK").is_empty() || !si.env("LLAMA_ARG_NO_MMAP").is_empty())
|
||||
{
|
||||
let mut m = HashMap::new();
|
||||
m.insert(
|
||||
"LLAMA_ARG_LOAD_MODE".to_string(),
|
||||
si.env("LLAMA_ARG_LOAD_MODE").to_string(),
|
||||
);
|
||||
restricted = m;
|
||||
&restricted
|
||||
} else {
|
||||
&si.arg_env
|
||||
};
|
||||
let mode = resident_load_mode(args, env);
|
||||
let model_mib = si.model_bytes >> 20;
|
||||
if mode.is_empty() || model_mib <= 0 || si.ram_mib <= 0 {
|
||||
return (String::new(), String::new());
|
||||
}
|
||||
let remote = has_any_flag(args, &["--rpc"]) || !trim(si.env("LLAMA_ARG_RPC")).is_empty();
|
||||
let (mut low, mut est, mut up_to, mut unsure) = (0i64, model_mib, true, false);
|
||||
if remote {
|
||||
} else if si.unified_mem {
|
||||
low = model_mib;
|
||||
est = model_mib;
|
||||
up_to = false;
|
||||
} else if si.engine_vram_mib > 0 {
|
||||
low = (model_mib - si.engine_vram_mib).max(0);
|
||||
est = low;
|
||||
up_to = false;
|
||||
} else if si.vram_mib > 0 {
|
||||
est = (model_mib - si.vram_mib).max(0);
|
||||
up_to = false;
|
||||
unsure = true;
|
||||
}
|
||||
let ram = si.ram_mib;
|
||||
let what = if up_to { "jusqu'à " } else { "environ " };
|
||||
if unsure && est * 10 > ram * 9 && !load_guard_off(cfg) {
|
||||
return (
|
||||
format!(
|
||||
"{mode} : environ {} de poids en RAM (modèle {}, VRAM NVIDIA {}) pour {} — échec probable ; refus seulement si les cartes listées par le moteur le confirment, lancement sinon ; --load-mode mmap est plus sûr",
|
||||
gib_text(est),
|
||||
gib_text(model_mib),
|
||||
gib_text(si.vram_mib),
|
||||
gib_text(ram)
|
||||
),
|
||||
String::new(),
|
||||
);
|
||||
}
|
||||
if low * 10 > ram * 9 {
|
||||
let msg = format!(
|
||||
"{mode} : au moins {} de poids restent en RAM (modèle {}) pour {} de RAM, limite du conteneur comprise — échec certain : swap et décodage effondré, ou processus tué (llama.cpp #26110). --load-mode mmap laisse le système relire les pages au besoin",
|
||||
gib_text(low),
|
||||
gib_text(model_mib),
|
||||
gib_text(ram)
|
||||
);
|
||||
if load_guard_off(cfg) {
|
||||
return (msg + " ; LOAD_GUARD=off : lancé quand même", String::new());
|
||||
}
|
||||
return (String::new(), msg + " ; " + LOAD_GUARD_MARKER);
|
||||
}
|
||||
if est * 10 > ram * 8 {
|
||||
return (
|
||||
format!(
|
||||
"{mode} : {what}{} de poids en RAM pour {} — au-delà, swap (décodage de 25 à 7,5 t/s dans llama.cpp #26110) ou processus tué ; --load-mode mmap est plus sûr",
|
||||
gib_text(est),
|
||||
gib_text(ram)
|
||||
),
|
||||
String::new(),
|
||||
);
|
||||
}
|
||||
(String::new(), String::new())
|
||||
}
|
||||
|
||||
/// `loadGuardNeedsDevices`.
|
||||
pub fn load_guard_needs_devices(cfg: &Cfg, args: &[String], si: &SysInfo) -> bool {
|
||||
if si.engine_vram_mib > 0 || si.vram_mib <= 0 {
|
||||
return false;
|
||||
}
|
||||
let mut s2 = si.clone();
|
||||
s2.engine_vram_mib = si.vram_mib;
|
||||
!load_mode_risk(cfg, args, &s2).1.is_empty()
|
||||
}
|
||||
|
||||
/// `engineDevsVRAM`.
|
||||
pub fn engine_devs_vram(devs: &[SplitDev]) -> i64 {
|
||||
let mut total = 0;
|
||||
for d in devs {
|
||||
if d.total_mib <= 0 {
|
||||
return 0;
|
||||
}
|
||||
total += d.total_mib;
|
||||
}
|
||||
total
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
//! État du slot gardé à travers une bascule de preset
|
||||
//! (`backend_serve_persist.go`, `SLOT_PERSIST`, off par défaut) : la clé du
|
||||
//! moteur (empreinte de tout ce qui décide du calcul du cache), le marqueur
|
||||
//! lu par le process web, le ménage des états gardés.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, cpu_weights, flag_value, get, has_any_flag, is_external, loopback_host,
|
||||
spec_mode, spec_user_set, split_mode_key, trim,
|
||||
};
|
||||
|
||||
/// `slotPersistOn`.
|
||||
pub fn slot_persist_on(cfg: &Cfg) -> bool {
|
||||
matches!(
|
||||
trim(get(cfg, "SLOT_PERSIST")).to_lowercase().as_str(),
|
||||
"on" | "1" | "true" | "yes" | "oui"
|
||||
)
|
||||
}
|
||||
|
||||
/// `slotPersistArgEnv`.
|
||||
pub const SLOT_PERSIST_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_SLOT_SAVE_PATH",
|
||||
"LLAMA_ARG_SPEC_TYPE",
|
||||
"LLAMA_ARG_SPEC_DRAFT_MODEL",
|
||||
"LLAMA_ARG_SPEC_DRAFT_HF_REPO",
|
||||
];
|
||||
|
||||
/// `slotPersistPlan` : (accepté, refus — vide = clé absente).
|
||||
pub fn slot_persist_plan(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (bool, String) {
|
||||
if !slot_persist_on(cfg) {
|
||||
return (false, String::new());
|
||||
}
|
||||
let no = |w: &str| (false, w.to_string());
|
||||
if is_external(cfg) {
|
||||
return no("preset externe");
|
||||
}
|
||||
if !si.help.contains("--slot-save-path") {
|
||||
return no("ce moteur ne connaît pas --slot-save-path");
|
||||
}
|
||||
if has_any_flag(extra, &["--slot-save-path"]) || !si.env("LLAMA_ARG_SLOT_SAVE_PATH").is_empty()
|
||||
{
|
||||
return no("--slot-save-path est déjà réglé à la main");
|
||||
}
|
||||
if si.slot_dir.is_empty() {
|
||||
return no("dossier des slots indisponible");
|
||||
}
|
||||
if spec_mode(cfg) != "off"
|
||||
|| spec_user_set(extra, &si.arg_env)
|
||||
|| !si.env("LLAMA_ARG_SPEC_TYPE").is_empty()
|
||||
{
|
||||
return no(
|
||||
"décodage spéculatif : la sauvegarde du moteur ne garde pas l'état du brouillon",
|
||||
);
|
||||
}
|
||||
if !split_mode_key(cfg).is_empty() {
|
||||
return no("SPLIT_MODE : état réparti entre cartes, sauvegarde non validée");
|
||||
}
|
||||
if !slot_save_safe(cfg, extra, si) {
|
||||
return no(
|
||||
"moteur joignable par d'autres sans clé d'API (HOST=127.0.0.1 ou une clé d'API)",
|
||||
);
|
||||
}
|
||||
let r = cpu_weights(cfg, extra, &si.arg_env, si.blocks());
|
||||
if !r.is_empty() {
|
||||
return (false, r);
|
||||
}
|
||||
(true, String::new())
|
||||
}
|
||||
|
||||
/// `slotSaveSafe` : boucle locale, ou clé d'API exigée.
|
||||
pub fn slot_save_safe(cfg: &Cfg, extra: &[String], si: &SysInfo) -> bool {
|
||||
let mut host = flag_value(extra, &["--host"]);
|
||||
if host.is_empty() {
|
||||
host = get(cfg, "HOST").into();
|
||||
}
|
||||
if host.is_empty() {
|
||||
host = "0.0.0.0".into();
|
||||
}
|
||||
loopback_host(&host) || !si.api_key.is_empty()
|
||||
}
|
||||
|
||||
/// `slotPersistNote`.
|
||||
pub fn slot_persist_note(ok: bool, why: &str) -> String {
|
||||
if ok || why.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
format!("SLOT_PERSIST refusé ({why}) : l'état du slot ne sera pas gardé à la bascule de preset")
|
||||
}
|
||||
|
||||
/// `slotPersistFileRe`.
|
||||
pub fn slot_persist_file(name: &str) -> bool {
|
||||
let Some(rest) = name.strip_prefix("persist-") else {
|
||||
return false;
|
||||
};
|
||||
let Some(rest) = rest.strip_suffix(".bin") else {
|
||||
return false;
|
||||
};
|
||||
let hex = |s: &str, n: usize| {
|
||||
s.len() == n && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
|
||||
};
|
||||
match rest.split_once('-') {
|
||||
Some((a, b)) => hex(a, 16) && hex(b, 12),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub const SLOT_PERSIST_MAX_FILES: usize = 2;
|
||||
|
||||
fn mod_nanos(m: &std::fs::Metadata) -> i128 {
|
||||
m.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map_or(0, |d| d.as_nanos() as i128)
|
||||
}
|
||||
|
||||
/// `slotPersistKey` : empreinte (16 hex) de la ligne finale (sans la clé
|
||||
/// d'API), des variables LLAMA_ARG_/GGML_/CUDA_ (environnement puis `env`),
|
||||
/// des fichiers (taille, date) et du build.
|
||||
pub fn slot_persist_key(
|
||||
args: &[String],
|
||||
env: &std::collections::BTreeMap<String, String>,
|
||||
proc_env: &[(String, String)],
|
||||
si: &SysInfo,
|
||||
bin: &str,
|
||||
) -> String {
|
||||
let mut h = Sha256::new();
|
||||
let mut w = |s: &str| {
|
||||
h.update(s.as_bytes());
|
||||
h.update([0u8]);
|
||||
};
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
if args[i] == "--api-key" {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
w(&args[i]);
|
||||
i += 1;
|
||||
}
|
||||
let mut vars = std::collections::BTreeMap::new();
|
||||
for (k, v) in proc_env {
|
||||
if k.starts_with("LLAMA_ARG_") || k.starts_with("GGML_") || k.starts_with("CUDA_") {
|
||||
vars.insert(k.clone(), v.clone());
|
||||
}
|
||||
}
|
||||
for (k, v) in env {
|
||||
vars.insert(k.clone(), v.clone());
|
||||
}
|
||||
vars.remove("LLAMA_ARG_API_KEY");
|
||||
for (k, v) in &vars {
|
||||
w(&format!("{k}={v}"));
|
||||
}
|
||||
let mut stat = |p: &str| {
|
||||
if p.is_empty() {
|
||||
return;
|
||||
}
|
||||
match std::fs::metadata(p) {
|
||||
Ok(m) => w(&format!("{p}|{}|{}", m.len(), mod_nanos(&m))),
|
||||
Err(_) => w(&format!("{p}|?")),
|
||||
}
|
||||
};
|
||||
if !si.model.is_empty() {
|
||||
let mp = Path::new(&si.model);
|
||||
let dir = mp.parent().unwrap_or(Path::new("."));
|
||||
let base = mp
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
for n in crate::gguf::shard_family(&base) {
|
||||
stat(&dir.join(n).to_string_lossy());
|
||||
}
|
||||
stat(&si.model);
|
||||
}
|
||||
stat(&si.mmproj);
|
||||
stat(&si.draft);
|
||||
stat(bin);
|
||||
let bdir = Path::new(bin).parent().unwrap_or(Path::new("."));
|
||||
if let Ok(rd) = std::fs::read_dir(bdir) {
|
||||
let mut names: Vec<String> = rd
|
||||
.flatten()
|
||||
.map(|e| e.file_name().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
names.sort();
|
||||
for name in names {
|
||||
let n = name.to_lowercase();
|
||||
if n.ends_with(".so")
|
||||
|| n.contains(".so.")
|
||||
|| n.ends_with(".dll")
|
||||
|| n.ends_with(".dylib")
|
||||
{
|
||||
stat(&bdir.join(&name).to_string_lossy());
|
||||
}
|
||||
}
|
||||
}
|
||||
w(&format!("build={}", si.engine_build));
|
||||
let d = h.finalize();
|
||||
d.iter().map(|b| format!("{b:02x}")).collect::<String>()[..16].to_string()
|
||||
}
|
||||
|
||||
/// `slotPersistMarker`.
|
||||
pub fn slot_persist_marker(home: &Path) -> PathBuf {
|
||||
home.join("slots-engine.key")
|
||||
}
|
||||
|
||||
/// `writeSlotPersistMarker`.
|
||||
pub fn write_slot_persist_marker(home: &Path, key: &str) {
|
||||
let p = slot_persist_marker(home);
|
||||
if key.is_empty() {
|
||||
let _ = std::fs::remove_file(p);
|
||||
return;
|
||||
}
|
||||
let _ = super::run::write_private(&p, format!("{key}\n").as_bytes());
|
||||
}
|
||||
|
||||
/// `slotPersistPrune` : garde les deux états les plus récents.
|
||||
pub fn slot_persist_prune(dir: &Path) {
|
||||
let Ok(rd) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
let mut fs: Vec<(String, i128)> = rd
|
||||
.flatten()
|
||||
.filter(|e| e.file_type().is_ok_and(|t| t.is_file()))
|
||||
.filter_map(|e| {
|
||||
let n = e.file_name().to_string_lossy().into_owned();
|
||||
if !slot_persist_file(&n) {
|
||||
return None;
|
||||
}
|
||||
e.metadata().ok().map(|m| (n, mod_nanos(&m)))
|
||||
})
|
||||
.collect();
|
||||
fs.sort_by_key(|f| std::cmp::Reverse(f.1));
|
||||
for (n, _) in fs.iter().skip(SLOT_PERSIST_MAX_FILES) {
|
||||
let _ = std::fs::remove_file(dir.join(n));
|
||||
}
|
||||
}
|
||||
|
||||
/// `slotPersistPurge`.
|
||||
pub fn slot_persist_purge(dir: &Path) {
|
||||
let Ok(rd) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for e in rd.flatten() {
|
||||
let n = e.file_name().to_string_lossy().into_owned();
|
||||
if e.file_type().is_ok_and(|t| t.is_file()) && slot_persist_file(&n) {
|
||||
let _ = std::fs::remove_file(dir.join(n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `prepareSlotDir` : LOKI_HOME/slots (0700) vidé, sauf les états gardés
|
||||
/// quand SLOT_PERSIST est demandé ; vide = indisponible.
|
||||
pub fn prepare_slot_dir(home: &Path, keep_persist: bool) -> String {
|
||||
let dir = home.join("slots");
|
||||
let mut b = std::fs::DirBuilder::new();
|
||||
b.recursive(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
// MkdirAll(0700) : seuls les dossiers créés reçoivent le mode.
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
b.mode(0o700);
|
||||
}
|
||||
if b.create(&dir).is_err() {
|
||||
return String::new();
|
||||
}
|
||||
match std::fs::metadata(&dir) {
|
||||
Ok(m) if m.is_dir() => {}
|
||||
_ => return String::new(),
|
||||
}
|
||||
if let Ok(rd) = std::fs::read_dir(&dir) {
|
||||
for e in rd.flatten() {
|
||||
let n = e.file_name().to_string_lossy().into_owned();
|
||||
if e.file_type().is_ok_and(|t| t.is_file()) && !(keep_persist && slot_persist_file(&n))
|
||||
{
|
||||
let _ = std::fs::remove_file(dir.join(n));
|
||||
}
|
||||
}
|
||||
}
|
||||
dir.to_string_lossy().into_owned()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
//! Files de lancement CUDA sur deux GPU ou plus
|
||||
//! (`backend_serve_queues.go`) : `CUDA_SCALE_LAUNCH_QUEUES=4x` là où le
|
||||
//! pipeline entre cartes peut exister.
|
||||
|
||||
use super::{Cfg, SysInfo, env_truthy, flag_value, flag_values, get, has_any_flag, trim};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub const LAUNCH_QUEUES_AUTO: &str = "4x";
|
||||
|
||||
/// `serveArgEnv`.
|
||||
pub const SERVE_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_DEVICE",
|
||||
"LLAMA_ARG_SPLIT_MODE",
|
||||
"LLAMA_ARG_N_GPU_LAYERS",
|
||||
"LLAMA_ARG_OVERRIDE_TENSOR",
|
||||
"LLAMA_ARG_CPU_MOE",
|
||||
"LLAMA_ARG_N_CPU_MOE",
|
||||
"LLAMA_ARG_N_CPU_FFN",
|
||||
"LLAMA_ARG_KV_OFFLOAD",
|
||||
"LLAMA_ARG_NO_KV_OFFLOAD",
|
||||
];
|
||||
|
||||
fn launch_queue_value(v: &str) -> bool {
|
||||
matches!(v, "0.25x" | "0.5x" | "2x" | "4x")
|
||||
}
|
||||
|
||||
/// `launchQueuesEnv` : valeur vide = ne rien poser.
|
||||
pub fn launch_queues_env(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (String, Vec<String>) {
|
||||
if !si.launch_queues.is_empty() {
|
||||
return (String::new(), Vec::new());
|
||||
}
|
||||
let mut notes = Vec::new();
|
||||
let want = trim(get(cfg, "CUDA_LAUNCH_QUEUES")).to_lowercase();
|
||||
if want == "off" {
|
||||
return (String::new(), notes);
|
||||
}
|
||||
if launch_queue_value(&want) {
|
||||
return (
|
||||
want.clone(),
|
||||
vec![format!(
|
||||
"CUDA_SCALE_LAUNCH_QUEUES={want} (CUDA_LAUNCH_QUEUES)"
|
||||
)],
|
||||
);
|
||||
}
|
||||
if !want.is_empty() && want != "auto" {
|
||||
notes.push(format!(
|
||||
"CUDA_LAUNCH_QUEUES={} ignoré (0.25x, 0.5x, 2x, 4x ou off) → automatique",
|
||||
get(cfg, "CUDA_LAUNCH_QUEUES")
|
||||
));
|
||||
}
|
||||
let n = served_gpu_count(extra, si);
|
||||
if n < 2 || !pipeline_blocker(cfg, extra, &si.arg_env).is_empty() {
|
||||
return (String::new(), notes);
|
||||
}
|
||||
notes.push(format!(
|
||||
"CUDA_SCALE_LAUNCH_QUEUES={LAUNCH_QUEUES_AUTO} ({n} GPU, parallélisme pipeline possible) — CUDA_LAUNCH_QUEUES=off pour s'en passer"
|
||||
));
|
||||
(LAUNCH_QUEUES_AUTO.into(), notes)
|
||||
}
|
||||
|
||||
/// `servedGPUCount`.
|
||||
pub fn served_gpu_count(extra: &[String], si: &SysInfo) -> i64 {
|
||||
let mut dev = flag_value(extra, &["-dev", "--device"]);
|
||||
if dev.is_empty() {
|
||||
dev = si.env("LLAMA_ARG_DEVICE").to_string();
|
||||
}
|
||||
if dev.is_empty() {
|
||||
return si.gpus;
|
||||
}
|
||||
dev.split(',')
|
||||
.filter(|d| trim(d).to_uppercase().starts_with("CUDA"))
|
||||
.count() as i64
|
||||
}
|
||||
|
||||
/// `pipelineBlocker` : pourquoi llama.cpp n'ouvrira PAS le pipeline (vide =
|
||||
/// il peut).
|
||||
pub fn pipeline_blocker(cfg: &Cfg, extra: &[String], arg_env: &HashMap<String, String>) -> String {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
let r = tensor_override(extra, arg_env);
|
||||
if !r.is_empty() {
|
||||
return r;
|
||||
}
|
||||
if !kv_offloaded(extra, arg_env) {
|
||||
return "cache KV sur CPU".into();
|
||||
}
|
||||
let mut sm = flag_value(extra, &["-sm", "--split-mode"]);
|
||||
if sm.is_empty() {
|
||||
sm = env("LLAMA_ARG_SPLIT_MODE").into();
|
||||
}
|
||||
if !sm.is_empty() && sm != "layer" {
|
||||
return format!("découpe {sm}");
|
||||
}
|
||||
let mut ngl = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if ngl.is_empty() {
|
||||
ngl = trim(get(cfg, "NGL")).to_string();
|
||||
if ngl.eq_ignore_ascii_case("auto") {
|
||||
ngl = env("LLAMA_ARG_N_GPU_LAYERS").into();
|
||||
}
|
||||
}
|
||||
let ngl = trim(&ngl).to_lowercase();
|
||||
if !ngl.is_empty() && ngl != "auto" && ngl != "all" && ngl != "999" {
|
||||
return format!("couches GPU limitées à {ngl}");
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `tensorOverride` : tenseurs placés à la main (vide = aucun).
|
||||
pub fn tensor_override(extra: &[String], arg_env: &HashMap<String, String>) -> String {
|
||||
let env = |k: &str| arg_env.get(k).map_or("", String::as_str);
|
||||
if has_any_flag(extra, &["-ot", "--override-tensor", "--cpu-moe", "-cmoe"])
|
||||
|| !env("LLAMA_ARG_OVERRIDE_TENSOR").is_empty()
|
||||
|| env_truthy(env("LLAMA_ARG_CPU_MOE"))
|
||||
{
|
||||
return "surcharge de tenseurs".into();
|
||||
}
|
||||
for (e, short, long) in [
|
||||
("LLAMA_ARG_N_CPU_MOE", "-ncmoe", "--n-cpu-moe"),
|
||||
("LLAMA_ARG_N_CPU_FFN", "-ncffn", "--n-cpu-ffn"),
|
||||
] {
|
||||
let mut vals = flag_values(extra, &[short, long]);
|
||||
vals.push(env(e).to_string());
|
||||
for v in vals {
|
||||
let v = trim(&v);
|
||||
if !v.is_empty() && v != "0" {
|
||||
return format!("couches {long} sur CPU");
|
||||
}
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `kvOffloaded` : le cache KV reste-t-il sur GPU ?
|
||||
pub fn kv_offloaded(extra: &[String], arg_env: &HashMap<String, String>) -> bool {
|
||||
let (mut on, mut set) = (true, false);
|
||||
for a in extra {
|
||||
let name = a.split_once('=').map_or(a.as_str(), |(n, _)| n);
|
||||
match name {
|
||||
"-kvo" | "--kv-offload" => {
|
||||
on = true;
|
||||
set = true;
|
||||
}
|
||||
"-nkvo" | "--no-kv-offload" => {
|
||||
on = false;
|
||||
set = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if set {
|
||||
return on;
|
||||
}
|
||||
if arg_env
|
||||
.get("LLAMA_ARG_NO_KV_OFFLOAD")
|
||||
.is_some_and(|v| !v.is_empty())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
!matches!(
|
||||
trim(
|
||||
arg_env
|
||||
.get("LLAMA_ARG_KV_OFFLOAD")
|
||||
.map_or("", String::as_str)
|
||||
)
|
||||
.to_lowercase()
|
||||
.as_str(),
|
||||
"off" | "disabled" | "false" | "0"
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,884 @@
|
||||
//! `loki-rs engine serve` — port de `cmdServe` : lit le monde (configuration
|
||||
//! par l'API interne du Go, aide du moteur, GGUF, RAM, VRAM, cgroup,
|
||||
//! variables), compose la ligne par [`super::build_serve_args`], puis cède la
|
||||
//! place à llama-server (`exec`).
|
||||
//!
|
||||
//! Ce qui reste au Go : un preset Strata et le moteur d'essai de
|
||||
//! l'optimiseur (`LOKI_TUNE_TRIAL`) — `loki serve` est alors lancé à notre
|
||||
//! place, à l'identique. Sans accès à l'API interne (lancement à la main hors
|
||||
//! de la façade), même repli.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Write as _;
|
||||
use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::*;
|
||||
use crate::gguf;
|
||||
use crate::store::{BK_CONFIG, BK_STATE, Store};
|
||||
|
||||
/// Variables d'environnement du lancement : celles du process, plus ce que
|
||||
/// `loki serve` pose (sélection GPU, bibliothèques, variables du moteur).
|
||||
#[derive(Default)]
|
||||
pub struct Env {
|
||||
overlay: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Env {
|
||||
pub fn get(&self, k: &str) -> Option<String> {
|
||||
self.overlay
|
||||
.get(k)
|
||||
.cloned()
|
||||
.or_else(|| std::env::var(k).ok())
|
||||
}
|
||||
/// `os.Getenv` : vide si absente.
|
||||
pub fn getenv(&self, k: &str) -> String {
|
||||
self.get(k).unwrap_or_default()
|
||||
}
|
||||
pub fn set(&mut self, k: &str, v: &str) {
|
||||
self.overlay.insert(k.into(), v.into());
|
||||
}
|
||||
/// L'environnement complet, trié.
|
||||
pub fn all(&self) -> Vec<(String, String)> {
|
||||
let mut m: BTreeMap<String, String> = std::env::vars().collect();
|
||||
for (k, v) in &self.overlay {
|
||||
m.insert(k.clone(), v.clone());
|
||||
}
|
||||
m.into_iter().collect()
|
||||
}
|
||||
fn command(&self, bin: &str) -> Command {
|
||||
let mut c = Command::new(bin);
|
||||
c.envs(&self.overlay);
|
||||
c
|
||||
}
|
||||
}
|
||||
|
||||
/// Écrit un fichier privé (0600).
|
||||
pub(crate) fn write_private(p: &Path, data: &[u8]) -> std::io::Result<()> {
|
||||
let mut o = std::fs::OpenOptions::new();
|
||||
o.write(true).create(true).truncate(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
o.mode(0o600);
|
||||
}
|
||||
o.open(p)?.write_all(data)
|
||||
}
|
||||
|
||||
/// Sortie combinée d'une commande bornée dans le temps ; `None` si elle n'a
|
||||
/// pas pu démarrer. (succès, sortie)
|
||||
fn output_with_timeout(mut cmd: Command, timeout: Duration) -> Option<(bool, String)> {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"loki-serve-{}-{}.out",
|
||||
std::process::id(),
|
||||
Instant::now().elapsed().as_nanos() ^ rand_u64()
|
||||
));
|
||||
let f = std::fs::File::create(&path).ok()?;
|
||||
let f2 = f.try_clone().ok()?;
|
||||
let mut child = cmd.stdin(Stdio::null()).stdout(f).stderr(f2).spawn().ok()?;
|
||||
let deadline = Instant::now() + timeout;
|
||||
let ok = loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(st)) => break st.success(),
|
||||
Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(20)),
|
||||
_ => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
break false;
|
||||
}
|
||||
}
|
||||
};
|
||||
let out = std::fs::read(&path).unwrap_or_default();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
Some((ok, String::from_utf8_lossy(&out).into_owned()))
|
||||
}
|
||||
|
||||
fn rand_u64() -> u128 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_nanos())
|
||||
}
|
||||
|
||||
/// `binHelp` : sortie de `<bin> --help` (vide si illisible).
|
||||
pub fn bin_help(env: &Env, bin: &str) -> String {
|
||||
let mut c = env.command(bin);
|
||||
c.arg("--help");
|
||||
match output_with_timeout(c, Duration::from_secs(5)) {
|
||||
Some((ok, out)) if ok || !out.is_empty() => out,
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `libraryPathValue` (Linux) : dossier du binaire, CUDA, puis l'existant.
|
||||
pub fn library_path_value(env: &Env, dir: &str) -> (String, String) {
|
||||
let mut parts = vec![dir.to_string()];
|
||||
parts.extend(cuda_lib_dirs());
|
||||
let var = if cfg!(target_os = "macos") {
|
||||
"DYLD_LIBRARY_PATH"
|
||||
} else {
|
||||
"LD_LIBRARY_PATH"
|
||||
};
|
||||
let existing = env.getenv(var);
|
||||
if !existing.is_empty() {
|
||||
parts.push(existing);
|
||||
}
|
||||
(var.into(), parts.join(":"))
|
||||
}
|
||||
|
||||
fn is_dir(p: &str) -> bool {
|
||||
std::fs::metadata(p).is_ok_and(|m| m.is_dir())
|
||||
}
|
||||
|
||||
fn is_file(p: &str) -> bool {
|
||||
std::fs::metadata(p).is_ok_and(|m| !m.is_dir())
|
||||
}
|
||||
|
||||
/// `cudaLibDirs`.
|
||||
fn cuda_lib_dirs() -> Vec<String> {
|
||||
let mut dirs: Vec<String> = Vec::new();
|
||||
let mut add = |d: String| {
|
||||
if !d.is_empty() && !dirs.contains(&d) && is_dir(&d) {
|
||||
dirs.push(d);
|
||||
}
|
||||
};
|
||||
add("/usr/local/cuda/lib64".into());
|
||||
add("/usr/local/cuda/targets/x86_64-linux/lib".into());
|
||||
let mut versioned: Vec<String> = std::fs::read_dir("/usr/local")
|
||||
.map(|rd| {
|
||||
rd.flatten()
|
||||
.map(|e| e.file_name().to_string_lossy().into_owned())
|
||||
.filter(|n| n.starts_with("cuda-"))
|
||||
.map(|n| format!("/usr/local/{n}/lib64"))
|
||||
.filter(|p| Path::new(p).exists())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
versioned.sort();
|
||||
for v in versioned.into_iter().rev() {
|
||||
add(v);
|
||||
}
|
||||
dirs
|
||||
}
|
||||
|
||||
/// `engineSource` / `prebuiltOwns` / `engineOwns` / `samePath`.
|
||||
pub struct EnginePaths {
|
||||
pub home: PathBuf,
|
||||
}
|
||||
|
||||
fn norm_path(s: &str) -> String {
|
||||
crate::paths::go_clean(s).replace('\\', "/").to_lowercase()
|
||||
}
|
||||
|
||||
impl EnginePaths {
|
||||
fn prebuilt_dir(&self) -> String {
|
||||
self.home
|
||||
.join("backends")
|
||||
.join("llama.cpp-prebuilt")
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
}
|
||||
fn engine_dir(&self) -> String {
|
||||
self.home.join("engine").to_string_lossy().into_owned()
|
||||
}
|
||||
pub fn prebuilt_owns(&self, p: &str) -> bool {
|
||||
!p.is_empty() && norm_path(p).starts_with(&(norm_path(&self.prebuilt_dir()) + "/"))
|
||||
}
|
||||
pub fn engine_owns(&self, p: &str) -> bool {
|
||||
!p.is_empty() && norm_path(p).starts_with(&(norm_path(&self.engine_dir()) + "/"))
|
||||
}
|
||||
/// `prebuiltServerBin` : le binaire du VERSION, sinon le dernier.
|
||||
fn prebuilt_server_bin(&self) -> String {
|
||||
let mut found = Vec::new();
|
||||
fn walk(d: &Path, out: &mut Vec<String>) {
|
||||
let Ok(rd) = std::fs::read_dir(d) else { return };
|
||||
for e in rd.flatten() {
|
||||
let p = e.path();
|
||||
if e.file_type().is_ok_and(|t| t.is_dir()) {
|
||||
walk(&p, out);
|
||||
} else if e.file_name() == "llama-server" {
|
||||
out.push(p.to_string_lossy().into_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(Path::new(&self.prebuilt_dir()), &mut found);
|
||||
found.sort();
|
||||
if found.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let tag = std::fs::read_to_string(Path::new(&self.prebuilt_dir()).join("VERSION"))
|
||||
.ok()
|
||||
.and_then(|s| s.split_whitespace().next().map(str::to_string))
|
||||
.unwrap_or_default();
|
||||
if !tag.is_empty()
|
||||
&& let Some(p) = found
|
||||
.iter()
|
||||
.find(|p| p.replace('\\', "/").contains(&format!("/llama-{tag}/")))
|
||||
{
|
||||
return p.clone();
|
||||
}
|
||||
found.last().cloned().unwrap_or_default()
|
||||
}
|
||||
/// `prebuiltResolveBin`.
|
||||
pub fn prebuilt_resolve_bin(&self, bin: &str) -> String {
|
||||
if !self.prebuilt_owns(bin) || std::fs::metadata(bin).is_ok() {
|
||||
return bin.into();
|
||||
}
|
||||
let cur = self.prebuilt_server_bin();
|
||||
if cur.is_empty() { bin.into() } else { cur }
|
||||
}
|
||||
/// `providedEngineBin`.
|
||||
fn provided_engine_bin(&self) -> String {
|
||||
let p = std::env::var("LOKI_ENGINE_BIN").unwrap_or_default();
|
||||
let p = trim(&p);
|
||||
if p.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
if is_file(&self.prebuilt_resolve_bin(p)) {
|
||||
p.into()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
/// `engineSource`.
|
||||
pub fn engine_source(&self, bin: &str) -> &'static str {
|
||||
let img = self.provided_engine_bin();
|
||||
if !img.is_empty() && same_path(bin, &img) {
|
||||
"image"
|
||||
} else if self.engine_owns(bin) {
|
||||
"downloaded"
|
||||
} else if self.prebuilt_owns(bin) {
|
||||
"prebuilt"
|
||||
} else {
|
||||
"custom"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `samePath` : symlinks résolus, casse ignorée.
|
||||
pub fn same_path(a: &str, b: &str) -> bool {
|
||||
if a.is_empty() || b.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let n = |p: &str| -> String {
|
||||
let r = std::fs::canonicalize(p)
|
||||
.map(|x| x.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|_| p.to_string());
|
||||
norm_path(&r)
|
||||
};
|
||||
n(a) == n(b)
|
||||
}
|
||||
|
||||
/// `parseEngineBuild`.
|
||||
pub fn parse_engine_build(out: &str) -> i64 {
|
||||
let long = regex::Regex::new(r"build\s+(\d+),\s*commit\s+([0-9a-fA-F]+)").expect("motif");
|
||||
let short = regex::Regex::new(r"(?m)^\s*(?:version|build)\s*:\s*(\d+)\s*\(([0-9a-fA-F]+)\)")
|
||||
.expect("motif");
|
||||
for re in [long, short] {
|
||||
if let Some(m) = re.captures(out) {
|
||||
return m[1].parse().unwrap_or(0);
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
/// `engineBuildTrusted`.
|
||||
fn engine_build_trusted(env: &Env, ep: &EnginePaths, bin: &str) -> i64 {
|
||||
let src = ep.engine_source(bin);
|
||||
if src == "custom" || !is_file(bin) {
|
||||
return 0;
|
||||
}
|
||||
let mut c = env.command(bin);
|
||||
c.arg("--version");
|
||||
let dir = Path::new(bin)
|
||||
.parent()
|
||||
.map(|d| d.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
let (k, v) = library_path_value(env, &dir);
|
||||
c.env(k, v);
|
||||
let out = output_with_timeout(c, Duration::from_secs(15))
|
||||
.map(|(_, o)| o)
|
||||
.unwrap_or_default();
|
||||
engine_build_trust(src, parse_engine_build(&out))
|
||||
}
|
||||
|
||||
fn has_tool(name: &str) -> bool {
|
||||
crate::paths::look_path(name).is_some()
|
||||
}
|
||||
|
||||
/// `nvidiaGPUCount`.
|
||||
fn nvidia_gpu_count() -> i64 {
|
||||
if !has_tool("nvidia-smi") {
|
||||
return 0;
|
||||
}
|
||||
let mut c = Command::new("nvidia-smi");
|
||||
c.args(["--query-gpu=index", "--format=csv,noheader"]);
|
||||
match output_with_timeout(c, Duration::from_secs(3)) {
|
||||
Some((true, out)) => out.split('\n').filter(|l| !trim(l).is_empty()).count() as i64,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// `nvidiaVRAMMiB`.
|
||||
fn nvidia_vram_mib(cvd: &str, pci: bool) -> i64 {
|
||||
if !has_tool("nvidia-smi") {
|
||||
return 0;
|
||||
}
|
||||
let mut c = Command::new("nvidia-smi");
|
||||
c.args([
|
||||
"--query-gpu=index,memory.total",
|
||||
"--format=csv,noheader,nounits",
|
||||
]);
|
||||
match output_with_timeout(c, Duration::from_secs(3)) {
|
||||
Some((true, out)) => vram_from_smi(&out, cvd, pci),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Lectures des variables du moteur (`probe*Env`).
|
||||
fn probe_env(env: &Env, si: &mut SysInfo, keys: &[&str], trim_check: bool) {
|
||||
for k in keys {
|
||||
if let Some(v) = env.get(k) {
|
||||
let keep = if trim_check {
|
||||
!trim(&v).is_empty()
|
||||
} else {
|
||||
!v.is_empty()
|
||||
};
|
||||
if keep {
|
||||
si.arg_env.insert((*k).into(), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `probeServeGPUs`.
|
||||
fn probe_serve_gpus(env: &Env, cfg: &Cfg, extra: &[String], si: &mut SysInfo) {
|
||||
si.launch_queues = env.getenv("CUDA_SCALE_LAUNCH_QUEUES");
|
||||
si.arg_env = HashMap::new();
|
||||
for k in SERVE_ARG_ENV {
|
||||
match env.get(k) {
|
||||
Some(v) if !v.is_empty() => {
|
||||
si.arg_env.insert((*k).into(), v);
|
||||
}
|
||||
Some(_) if *k == "LLAMA_ARG_NO_KV_OFFLOAD" => {
|
||||
si.arg_env.insert((*k).into(), "1".into());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if !si.launch_queues.is_empty()
|
||||
|| trim(get(cfg, "CUDA_LAUNCH_QUEUES")).eq_ignore_ascii_case("off")
|
||||
{
|
||||
return;
|
||||
}
|
||||
if has_any_flag(extra, &["-dev", "--device"]) || !si.env("LLAMA_ARG_DEVICE").is_empty() {
|
||||
return;
|
||||
}
|
||||
let cvd = trim(&env.getenv("CUDA_VISIBLE_DEVICES")).to_string();
|
||||
if !cvd.is_empty() {
|
||||
si.gpus = cvd.split(',').filter(|d| !trim(d).is_empty()).count() as i64;
|
||||
return;
|
||||
}
|
||||
si.gpus = nvidia_gpu_count();
|
||||
}
|
||||
|
||||
/// `probeCacheRAM`.
|
||||
fn probe_cache_ram(env: &Env, cfg: &Cfg, extra: &[String], si: &mut SysInfo) {
|
||||
probe_env(env, si, CACHE_ARG_ENV, true);
|
||||
if !si.model.is_empty() {
|
||||
let mp = Path::new(&si.model);
|
||||
let dir = mp.parent().unwrap_or(Path::new("."));
|
||||
let base = mp
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
si.model_bytes = crate::engstatus::shard_family_size(dir, &base);
|
||||
if let Ok(g) = gguf::gguf_meta(mp) {
|
||||
si.gguf = Some(g);
|
||||
}
|
||||
}
|
||||
let (used, total) = crate::sysinfo::ram_usage_mb();
|
||||
if total > 0 {
|
||||
si.ram_mib = total;
|
||||
si.ram_avail_mib = total - used;
|
||||
}
|
||||
if cfg!(target_os = "linux") {
|
||||
let (limit, cur) = cgroup_mem_mib(Path::new("/"));
|
||||
if limit > 0 {
|
||||
if si.ram_mib == 0 || limit < si.ram_mib {
|
||||
si.ram_mib = limit;
|
||||
}
|
||||
let free = limit - cur;
|
||||
if free >= 0 && (si.ram_avail_mib == 0 || free < si.ram_avail_mib) {
|
||||
si.ram_avail_mib = free;
|
||||
}
|
||||
}
|
||||
}
|
||||
si.unified_mem = cfg!(all(target_os = "macos", target_arch = "aarch64"));
|
||||
if cfg!(target_os = "macos")
|
||||
|| has_any_flag(extra, &["-dev", "--device"])
|
||||
|| !si.env("LLAMA_ARG_DEVICE").is_empty()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let (cvd, pci) = match get(cfg, "CUDA_VISIBLE_DEVICES") {
|
||||
"" => (
|
||||
env.getenv("CUDA_VISIBLE_DEVICES"),
|
||||
env.getenv("CUDA_DEVICE_ORDER") == "PCI_BUS_ID",
|
||||
),
|
||||
v => (v.to_string(), true),
|
||||
};
|
||||
si.vram_mib = nvidia_vram_mib(&cvd, pci);
|
||||
}
|
||||
|
||||
/// `listEngineDevices`.
|
||||
fn list_engine_devices(env: &Env, bin: &str) -> Vec<SplitDev> {
|
||||
let mut c = env.command(bin);
|
||||
c.arg("--list-devices");
|
||||
match output_with_timeout(c, Duration::from_secs(20)) {
|
||||
Some((true, out)) => parse_list_devices(&out),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `waitPortFree`.
|
||||
pub fn wait_port_free(host: &str, port: &str, wait: Duration) -> Result<(), String> {
|
||||
if port.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut h = host.trim_matches(['[', ']']).to_string();
|
||||
if h.is_empty() || h == "0.0.0.0" || h == "::" {
|
||||
h = "127.0.0.1".into();
|
||||
}
|
||||
let addr = if h.contains(':') {
|
||||
format!("[{h}]:{port}")
|
||||
} else {
|
||||
format!("{h}:{port}")
|
||||
};
|
||||
let deadline = Instant::now() + wait;
|
||||
loop {
|
||||
let reachable = addr
|
||||
.to_socket_addrs()
|
||||
.ok()
|
||||
.and_then(|mut a| a.next())
|
||||
.is_some_and(|sa: SocketAddr| {
|
||||
TcpStream::connect_timeout(&sa, Duration::from_millis(500)).is_ok()
|
||||
});
|
||||
if !reachable {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() > deadline {
|
||||
return Err(format!(
|
||||
"le port {port} est déjà utilisé (sans doute un ancien llama-server encore actif) : « loki stop », docker restart loki, ou change PORT"
|
||||
));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
/// Erreur de `loki serve` et son code de sortie.
|
||||
pub struct ServeError {
|
||||
pub msg: String,
|
||||
pub code: u8,
|
||||
}
|
||||
|
||||
impl From<String> for ServeError {
|
||||
fn from(msg: String) -> Self {
|
||||
ServeError { msg, code: 1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// Accès à la base du Go pour ce lancement.
|
||||
pub struct GoStore {
|
||||
rt: tokio::runtime::Runtime,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
impl GoStore {
|
||||
/// `LOKI_GO_ADDR` + `LOKI_INTERNAL_TOKEN`, posées par la façade pour le Go
|
||||
/// et héritées par ses enfants.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let addr: SocketAddr = std::env::var("LOKI_GO_ADDR").ok()?.parse().ok()?;
|
||||
let token = std::env::var(crate::auth::INTERNAL_ENV).ok()?;
|
||||
if token.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.ok()?;
|
||||
Some(GoStore {
|
||||
rt,
|
||||
store: Store::new(addr, token),
|
||||
})
|
||||
}
|
||||
pub fn config(&self) -> Result<Cfg, String> {
|
||||
self.rt
|
||||
.block_on(self.store.all(BK_CONFIG))
|
||||
.map(|m| m.into_iter().collect())
|
||||
}
|
||||
pub fn get(&self, bucket: &str, key: &str) -> Option<Vec<u8>> {
|
||||
self.rt.block_on(self.store.get(bucket, key)).ok().flatten()
|
||||
}
|
||||
pub fn put(&self, bucket: &str, key: &str, v: Vec<u8>) {
|
||||
let _ = self.rt.block_on(self.store.put(bucket, key, v));
|
||||
}
|
||||
pub fn del(&self, bucket: &str, key: &str) {
|
||||
let _ = self.rt.block_on(self.store.del(bucket, key));
|
||||
}
|
||||
}
|
||||
|
||||
/// Lit le jeton de tentative et les échecs inscrits.
|
||||
fn read_spec_auto(st: &GoStore) -> (Option<SpecAutoMark>, HashMap<String, String>) {
|
||||
let attempt = st
|
||||
.get(BK_STATE, SPEC_ATTEMPT_KEY)
|
||||
.and_then(|b| serde_json::from_slice::<SpecAutoMark>(&b).ok());
|
||||
let failed = st
|
||||
.get(BK_STATE, SPEC_FAILED_KEY)
|
||||
.and_then(|b| serde_json::from_slice::<HashMap<String, String>>(&b).ok())
|
||||
.unwrap_or_default();
|
||||
(attempt, failed)
|
||||
}
|
||||
|
||||
/// `specAutoSettle`.
|
||||
fn spec_auto_settle(st: &GoStore, cur: &SpecAutoMark, why: &str, record: bool, attempt: bool) {
|
||||
let (_, mut failed) = read_spec_auto(st);
|
||||
st.del(BK_STATE, SPEC_ATTEMPT_KEY);
|
||||
if record {
|
||||
if failed.len() >= SPEC_FAILED_MAX {
|
||||
failed.clear();
|
||||
}
|
||||
failed.insert(cur.key(), why.into());
|
||||
let sorted: BTreeMap<_, _> = failed.into_iter().collect();
|
||||
st.put(
|
||||
BK_STATE,
|
||||
SPEC_FAILED_KEY,
|
||||
crate::gojson::marshal(&sorted).into_bytes(),
|
||||
);
|
||||
}
|
||||
if attempt {
|
||||
st.put(
|
||||
BK_STATE,
|
||||
SPEC_ATTEMPT_KEY,
|
||||
crate::gojson::marshal(cur).into_bytes(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ce que [`prepare`] rend : de quoi lancer le moteur.
|
||||
pub struct Ready {
|
||||
pub bin: String,
|
||||
pub launch: Launch,
|
||||
pub env: Env,
|
||||
pub home: PathBuf,
|
||||
}
|
||||
|
||||
/// `cmdServe` jusqu'à la ligne de commande comprise (sans exec). `Ok(None)` :
|
||||
/// rien à lancer (preset externe).
|
||||
pub fn prepare(st: &GoStore, home: &Path) -> Result<Option<Ready>, ServeError> {
|
||||
let cfg = st.config()?;
|
||||
if is_external(&cfg) {
|
||||
println!("[info] preset externe actif : pas de moteur local à lancer");
|
||||
return Ok(None);
|
||||
}
|
||||
let mut env = Env::default();
|
||||
let mut bin = get(&cfg, "BIN").to_string();
|
||||
if bin.is_empty() {
|
||||
return Err("BIN non défini — lance « loki edit »".to_string().into());
|
||||
}
|
||||
let model = get(&cfg, "MODEL").to_string();
|
||||
if model.is_empty() {
|
||||
return Err("MODEL non défini — lance « loki edit »".to_string().into());
|
||||
}
|
||||
let stored =
|
||||
gguf::parse_stored_model_dirs(st.get(BK_STATE, gguf::STATE_MODEL_DIRS_KEY).as_deref());
|
||||
let env_dirs = std::env::var(gguf::MODEL_DIRS_ENV).ok();
|
||||
let resolve = |m: &str| gguf::resolve_serve_model_path(m, home, &stored, env_dirs.as_deref());
|
||||
let model = resolve(&model)?.to_string_lossy().into_owned();
|
||||
if std::fs::metadata(&model).is_err() {
|
||||
return Err(format!("modèle introuvable : {model}").into());
|
||||
}
|
||||
let mp = Path::new(&model);
|
||||
let mdir = mp.parent().unwrap_or(Path::new("."));
|
||||
let mbase = mp
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
let fam = gguf::shard_family(&mbase);
|
||||
let missing: Vec<String> = fam
|
||||
.iter()
|
||||
.filter(|n| !std::fs::metadata(mdir.join(n)).is_ok_and(|m| !m.is_dir()))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return Err(format!(
|
||||
"modèle incomplet : il manque {} dans {} — ce modèle tient en {} fichiers, télécharge-les tous",
|
||||
missing.join(", "),
|
||||
mdir.display(),
|
||||
fam.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if !crate::paths::go_is_abs(&bin) {
|
||||
bin = home.join(&bin).to_string_lossy().into_owned();
|
||||
}
|
||||
let ep = EnginePaths {
|
||||
home: home.to_path_buf(),
|
||||
};
|
||||
bin = ep.prebuilt_resolve_bin(&bin);
|
||||
let bdir = Path::new(&bin)
|
||||
.parent()
|
||||
.map(|d| d.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
let (lk, lv) = library_path_value(&env, &bdir);
|
||||
env.set(&lk, &lv);
|
||||
for (k, v) in cuda_device_env(&cfg) {
|
||||
env.set(&k, &v);
|
||||
}
|
||||
|
||||
let mut si = SysInfo {
|
||||
help: bin_help(&env, &bin),
|
||||
model: model.clone(),
|
||||
..SysInfo::default()
|
||||
};
|
||||
let mm = trim(get(&cfg, "MMPROJ")).to_string();
|
||||
if !mm.is_empty() {
|
||||
let p = resolve(&mm)
|
||||
.map_err(|e| format!("projecteur vision introuvable : {mm} ({e})"))?
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
if std::fs::metadata(&p).is_err() {
|
||||
return Err(format!("projecteur vision introuvable : {p}").into());
|
||||
}
|
||||
si.mmproj = p;
|
||||
}
|
||||
let key = st
|
||||
.get(BK_STATE, "api_key")
|
||||
.map(|b| String::from_utf8_lossy(&b).into_owned())
|
||||
.unwrap_or_default();
|
||||
si.api_key = if key.is_empty() {
|
||||
trim(get(&cfg, "API_KEY")).to_string()
|
||||
} else {
|
||||
key
|
||||
};
|
||||
if cfg!(target_os = "linux") && env.getenv("LLAMA_ARG_THREADS").is_empty() {
|
||||
si.cpu = cpuset_threads(Path::new("/"));
|
||||
}
|
||||
|
||||
let extra = crate::models::split_args(get(&cfg, "EXTRA_ARGS"));
|
||||
probe_serve_gpus(&env, &cfg, &extra, &mut si);
|
||||
// probeExpertEnv
|
||||
for k in EXPERT_ENV_KEYS {
|
||||
let v = env.getenv(k);
|
||||
if !v.is_empty() {
|
||||
si.user_env.insert((*k).into(), v);
|
||||
}
|
||||
}
|
||||
probe_env(&env, &mut si, FIT_ARG_ENV, false);
|
||||
// probeLoadEnv
|
||||
for k in LOAD_ARG_ENV {
|
||||
let v = trim(&env.getenv(k)).to_string();
|
||||
if !v.is_empty() {
|
||||
si.arg_env.insert((*k).into(), v);
|
||||
}
|
||||
}
|
||||
probe_env(&env, &mut si, FIDELITY_ARG_ENV, false);
|
||||
probe_cache_ram(&env, &cfg, &extra, &mut si);
|
||||
probe_env(&env, &mut si, CKPT_ARG_ENV, true);
|
||||
if side_slot_on(&cfg) {
|
||||
probe_env(&env, &mut si, SIDE_SLOT_ARG_ENV, true);
|
||||
}
|
||||
if split_mode_key(&cfg) == "tensor" && help_supports_split_tensor(&si.help) {
|
||||
probe_env(&env, &mut si, SPLIT_ARG_ENV, true);
|
||||
let ar = env.getenv("GGML_CUDA_ALLREDUCE");
|
||||
if !ar.is_empty() {
|
||||
si.user_env.insert("GGML_CUDA_ALLREDUCE".into(), ar);
|
||||
}
|
||||
si.split_devs = list_engine_devices(&env, &bin);
|
||||
}
|
||||
let (norm_extra, _) = normalize_load_flags(&extra, help_supports_load_mode(&si.help));
|
||||
if load_guard_needs_devices(&cfg, &norm_extra, &si) {
|
||||
let devs = if si.split_devs.is_empty() {
|
||||
list_engine_devices(&env, &bin)
|
||||
} else {
|
||||
si.split_devs.clone()
|
||||
};
|
||||
si.engine_vram_mib = engine_devs_vram(&devs);
|
||||
}
|
||||
let spec = spec_mode(&cfg);
|
||||
if spec != "off" && !spec.is_empty() {
|
||||
probe_env(&env, &mut si, SPEC_ARG_ENV, true);
|
||||
let d = trim(get(&cfg, "MODEL_DRAFT")).to_string();
|
||||
if !d.is_empty() {
|
||||
match resolve(&d)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|p| match std::fs::metadata(&p) {
|
||||
Ok(_) => Ok(p),
|
||||
Err(e) => Err(format!("stat {}: {e}", p.display())),
|
||||
}) {
|
||||
Ok(p) => {
|
||||
si.draft = p.to_string_lossy().into_owned();
|
||||
si.draft_is_model = same_file(&si.draft, &si.model);
|
||||
if let Ok(g) = gguf::gguf_meta(&p) {
|
||||
si.draft_gguf = Some(g);
|
||||
}
|
||||
}
|
||||
Err(e) => si.draft_err = e,
|
||||
}
|
||||
}
|
||||
}
|
||||
if gguf_hybrid(si.gguf.as_ref()) || spec == "auto" {
|
||||
si.engine_build = engine_build_trusted(&env, &ep, &bin);
|
||||
}
|
||||
let mark = SpecAutoMark {
|
||||
fp: crate::presets::config_fingerprint(&cfg),
|
||||
bin: bin.clone(),
|
||||
build: si.engine_build,
|
||||
};
|
||||
let mut spec_record = false;
|
||||
if spec == "auto" {
|
||||
let (attempt, failed) = read_spec_auto(st);
|
||||
let (why, rec) = spec_auto_verdict(&mark, attempt.as_ref(), &failed);
|
||||
si.spec_auto_blocked = why;
|
||||
spec_record = rec;
|
||||
}
|
||||
if slot_persist_on(&cfg) {
|
||||
probe_env(&env, &mut si, SLOT_PERSIST_ARG_ENV, true);
|
||||
}
|
||||
if si.help.contains("--slot-save-path") && !has_any_flag(&extra, &["--slot-save-path"]) {
|
||||
si.slot_dir = prepare_slot_dir(home, slot_persist_on(&cfg));
|
||||
}
|
||||
|
||||
let launch = build_serve_args(&cfg, &extra, &bin, &si);
|
||||
for (k, v) in &launch.env {
|
||||
env.set(k, v);
|
||||
}
|
||||
let mut persist_key = String::new();
|
||||
let (persist_ok, _) = slot_persist_plan(&cfg, &extra, &si);
|
||||
if persist_ok {
|
||||
persist_key = slot_persist_key(&launch.args, &launch.env, &env.all(), &si, &bin);
|
||||
slot_persist_prune(Path::new(&si.slot_dir));
|
||||
} else if !slot_persist_on(&cfg) {
|
||||
slot_persist_purge(&home.join("slots"));
|
||||
}
|
||||
for n in &launch.notes {
|
||||
eprintln!("[loki serve] {n}");
|
||||
}
|
||||
let (_, refuse) = load_mode_risk(&cfg, &launch.args, &si);
|
||||
if !refuse.is_empty() {
|
||||
return Err(ServeError {
|
||||
msg: refuse,
|
||||
code: LOAD_GUARD_EXIT_STATUS as u8,
|
||||
});
|
||||
}
|
||||
let host = arg_value(&launch.args, "--host");
|
||||
let port = arg_value(&launch.args, "--port");
|
||||
wait_port_free(&host, &port, Duration::from_secs(5))?;
|
||||
let (kt, vt, _) = effective_kv_types(&cfg, &extra, &si.arg_env);
|
||||
if slow_kv(&kt, &vt) {
|
||||
eprintln!(
|
||||
"[loki serve] avertissement : cache KV {kt}/{vt} non accéléré par llama.cpp CUDA (converti en f16 à chaque pas, lent). Préfère q8_0/q8_0, q4_0/q4_0 ou f16."
|
||||
);
|
||||
}
|
||||
if spec == "auto" {
|
||||
let auto = spec_args(&cfg, &extra, &si).auto;
|
||||
spec_auto_settle(st, &mark, &si.spec_auto_blocked, spec_record, auto);
|
||||
}
|
||||
write_slot_persist_marker(home, &persist_key);
|
||||
eprintln!(
|
||||
"[loki serve] {bin} model={} port={port}",
|
||||
crate::paths::base_name(&model)
|
||||
);
|
||||
st.put(
|
||||
BK_STATE,
|
||||
"engine_cmdline",
|
||||
engine_cmdline(&launch.env, &launch.args).into_bytes(),
|
||||
);
|
||||
Ok(Some(Ready {
|
||||
bin,
|
||||
launch,
|
||||
env,
|
||||
home: home.to_path_buf(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn same_file(a: &str, b: &str) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
match (std::fs::metadata(a), std::fs::metadata(b)) {
|
||||
(Ok(x), Ok(y)) => x.dev() == y.dev() && x.ino() == y.ino(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
same_path(a, b)
|
||||
}
|
||||
}
|
||||
|
||||
/// Lance `loki serve` (Go) à notre place : Strata, essai de l'optimiseur,
|
||||
/// pas d'API interne.
|
||||
fn delegate_to_go(go_bin: &Path) -> ExitCode {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::process::CommandExt;
|
||||
let err = Command::new(go_bin).arg("serve").exec();
|
||||
eprintln!("[err] exec {} serve : {err}", go_bin.display());
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
match Command::new(go_bin).arg("serve").status() {
|
||||
Ok(s) if s.success() => ExitCode::SUCCESS,
|
||||
_ => ExitCode::FAILURE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `loki-rs engine serve`.
|
||||
pub fn run(go_bin: &Path, home: &Path) -> ExitCode {
|
||||
if std::env::var("LOKI_TUNE_TRIAL").is_ok_and(|v| !v.is_empty()) {
|
||||
return delegate_to_go(go_bin);
|
||||
}
|
||||
let Some(st) = GoStore::from_env() else {
|
||||
return delegate_to_go(go_bin);
|
||||
};
|
||||
match st.config() {
|
||||
Ok(cfg) if is_strata(&cfg) => return delegate_to_go(go_bin),
|
||||
Ok(_) => {}
|
||||
Err(_) => return delegate_to_go(go_bin),
|
||||
}
|
||||
let ready = match prepare(&st, home) {
|
||||
Ok(Some(r)) => r,
|
||||
Ok(None) => return ExitCode::SUCCESS,
|
||||
Err(e) => {
|
||||
eprintln!("[err] {}", e.msg);
|
||||
return ExitCode::from(e.code);
|
||||
}
|
||||
};
|
||||
let _ = std::env::set_current_dir(&ready.home);
|
||||
exec_engine(&ready)
|
||||
}
|
||||
|
||||
fn exec_engine(r: &Ready) -> ExitCode {
|
||||
let args = &r.launch.args;
|
||||
let mut c = Command::new(&r.bin);
|
||||
c.args(&args[1..]).env_clear().envs(r.env.all());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::process::CommandExt;
|
||||
c.arg0(&args[0]);
|
||||
let err = c.exec();
|
||||
eprintln!("[err] exec {} : {err}", r.bin);
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
match c.status() {
|
||||
Ok(s) if s.success() => ExitCode::SUCCESS,
|
||||
_ => ExitCode::FAILURE,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Second slot pour les travaux annexes (`backend_serve_side.go`,
|
||||
//! `SIDE_SLOT`, off par défaut) : deux flux KV séparés de CTX jetons
|
||||
//! (`-c 2×CTX --parallel 2 --no-kv-unified`), refusé quand le compte de VRAM
|
||||
//! ne tient pas ou que l'utilisateur a réglé l'un des leviers.
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, atoi, cpu_weights, effective_kv_types, flag_value, get, gguf_state_bytes,
|
||||
has_any_flag, is_external, trim,
|
||||
};
|
||||
|
||||
/// `sideSlotOn`.
|
||||
pub fn side_slot_on(cfg: &Cfg) -> bool {
|
||||
matches!(
|
||||
trim(get(cfg, "SIDE_SLOT")).to_lowercase().as_str(),
|
||||
"on" | "1" | "true" | "yes" | "oui"
|
||||
)
|
||||
}
|
||||
|
||||
/// `sideSlotConflictFlags`.
|
||||
pub const SIDE_SLOT_CONFLICT_FLAGS: &[&str] = &[
|
||||
"-c",
|
||||
"--ctx-size",
|
||||
"-np",
|
||||
"--parallel",
|
||||
"-kvu",
|
||||
"--kv-unified",
|
||||
"-no-kvu",
|
||||
"--no-kv-unified",
|
||||
"--kv-unified-per-slot",
|
||||
"--cache-idle-slots",
|
||||
"--no-cache-idle-slots",
|
||||
];
|
||||
|
||||
/// `sideSlotArgEnv`.
|
||||
pub const SIDE_SLOT_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_N_PARALLEL",
|
||||
"LLAMA_ARG_KV_UNIFIED",
|
||||
"LLAMA_ARG_KV_UNIFIED_PER_SLOT",
|
||||
"LLAMA_ARG_CACHE_IDLE_SLOTS",
|
||||
"LLAMA_ARG_CTX_SIZE",
|
||||
];
|
||||
|
||||
/// `sideSlotPlan` : (jetons par slot, Mio de VRAM en plus, refus).
|
||||
pub fn side_slot_plan(cfg: &Cfg, extra: &[String], si: &SysInfo) -> (i64, i64, String) {
|
||||
if !side_slot_on(cfg) {
|
||||
return (0, 0, String::new());
|
||||
}
|
||||
if is_external(cfg) {
|
||||
return (0, 0, "preset externe".into());
|
||||
}
|
||||
let p = trim(get(cfg, "PARALLEL"));
|
||||
if !p.is_empty() && p != "2" {
|
||||
return (
|
||||
0,
|
||||
0,
|
||||
format!("PARALLEL={p} (SIDE_SLOT ouvre exactement 2 slots)"),
|
||||
);
|
||||
}
|
||||
for f in SIDE_SLOT_CONFLICT_FLAGS {
|
||||
if has_any_flag(extra, &[f]) {
|
||||
return (0, 0, format!("{f} est réglé dans EXTRA_ARGS"));
|
||||
}
|
||||
}
|
||||
for k in SIDE_SLOT_ARG_ENV {
|
||||
if !si.env(k).is_empty() {
|
||||
return (0, 0, format!("{k} est posée"));
|
||||
}
|
||||
}
|
||||
if !si.help.contains("--no-kv-unified") {
|
||||
return (0, 0, "ce moteur ne connaît pas --no-kv-unified".into());
|
||||
}
|
||||
let raw = trim(get(cfg, "CTX"));
|
||||
let n = if raw.is_empty() {
|
||||
Some(32768)
|
||||
} else {
|
||||
atoi(raw)
|
||||
};
|
||||
let n = match n {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => return (0, 0, "CTX doit être un nombre de jetons explicite".into()),
|
||||
};
|
||||
let (extra_mib, why) = side_slot_vram(cfg, extra, si, n);
|
||||
if !why.is_empty() {
|
||||
return (0, 0, why);
|
||||
}
|
||||
(n, extra_mib, String::new())
|
||||
}
|
||||
|
||||
/// `sideSlotVRAM`.
|
||||
pub fn side_slot_vram(cfg: &Cfg, extra: &[String], si: &SysInfo, ctx: i64) -> (i64, String) {
|
||||
if si.vram_mib <= 0 {
|
||||
return (0, "VRAM NVIDIA inconnue (macOS, AMD, --device…)".into());
|
||||
}
|
||||
let Some(g) = si.gguf.as_ref() else {
|
||||
return (0, "métadonnées GGUF illisibles".into());
|
||||
};
|
||||
if si.model_bytes <= 0 {
|
||||
return (0, "taille du modèle inconnue".into());
|
||||
}
|
||||
let r = cpu_weights(cfg, extra, &si.arg_env, g.block_count);
|
||||
if !r.is_empty() {
|
||||
return (0, r);
|
||||
}
|
||||
let (kt, vt, _) = effective_kv_types(cfg, extra, &si.arg_env);
|
||||
let (per_tok, rec) = gguf_state_bytes(g, &kt, &vt);
|
||||
if per_tok <= 0.0 {
|
||||
return (0, "taille d'état inconnue".into());
|
||||
}
|
||||
let state = per_tok * ctx as f64 + rec as f64;
|
||||
let mib = 1048576.0;
|
||||
if si.model_bytes as f64 + 2.0 * state > 0.9 * si.vram_mib as f64 * mib {
|
||||
return (
|
||||
0,
|
||||
format!(
|
||||
"VRAM insuffisante : modèle + 2 × {} Mio d'état dépassent 90 % de {} Mio",
|
||||
(state / mib).ceil() as i64,
|
||||
si.vram_mib
|
||||
),
|
||||
);
|
||||
}
|
||||
((state / mib).ceil() as i64, String::new())
|
||||
}
|
||||
|
||||
/// `sideSlotNote`.
|
||||
pub fn side_slot_note(ctx: i64, extra_mib: i64, why: &str, ngl_forced: bool) -> String {
|
||||
if ctx <= 0 {
|
||||
if why.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
return format!("SIDE_SLOT refusé ({why}) : un seul slot, comme sans la clé");
|
||||
}
|
||||
let n = format!(
|
||||
"SIDE_SLOT : 2 slots de {ctx} jetons (-c {}, --no-kv-unified), ~{extra_mib} Mio de VRAM en plus pour le second (cache KV et état récurrent ; le brouillon MTP éventuel et les points de reprise en RAM hôte doublent aussi)",
|
||||
2 * ctx
|
||||
);
|
||||
if ngl_forced {
|
||||
return n + " ; NGL est imposé : surveille la VRAM au premier long prompt";
|
||||
}
|
||||
n + " ; le placement automatique sortira des couches du GPU s'il manque de place"
|
||||
}
|
||||
|
||||
/// `nglForced` : le preset impose-t-il un nombre de couches GPU ?
|
||||
pub fn ngl_forced(cfg: &Cfg, extra: &[String]) -> bool {
|
||||
let mut v = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if v.is_empty() {
|
||||
v = trim(get(cfg, "NGL")).to_string();
|
||||
if v == "999" {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
!v.is_empty() && !v.eq_ignore_ascii_case("auto")
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
//! Décodage spéculatif géré par Loki (`backend_serve_spec.go`) : tête MTP,
|
||||
//! modèle brouillon, n-grammes du contexte. Rien n'y touche au modèle :
|
||||
//! chaque jeton émis est vérifié par le modèle cible.
|
||||
|
||||
use super::{
|
||||
Cfg, SysInfo, atoi, cpu_weights, fit_blocker, flag_value, flag_values, get, has_any_flag,
|
||||
has_flag_prefix, ngl_forced, op_offload_min_batch_env, trim,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub const SPEC_AUTO_MIN_BUILD: i64 = 11009;
|
||||
pub const SPEC_N_MAX_LIMIT: i64 = 64;
|
||||
pub const SPEC_NGRAM_N_MATCH: i64 = 24;
|
||||
pub const SPEC_NGRAM_N_MIN: i64 = 48;
|
||||
pub const SPEC_NGRAM_N_MAX: i64 = 64;
|
||||
pub const GGML_OFFLOAD_MIN_BATCH_DEFAULT: i64 = 32;
|
||||
|
||||
/// `specArgEnv`.
|
||||
pub const SPEC_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_SPEC_TYPE",
|
||||
"LLAMA_ARG_SPEC_DRAFT_MODEL",
|
||||
"LLAMA_ARG_SPEC_DRAFT_HF_REPO",
|
||||
"LLAMA_ARG_SPEC_DRAFT_N_MAX",
|
||||
"LLAMA_ARG_SPEC_DRAFT_SAMPLING",
|
||||
];
|
||||
|
||||
const SPEC_USER_FLAGS: &[&str] = &[
|
||||
"--spec-type",
|
||||
"--spec_type",
|
||||
"-md",
|
||||
"--model-draft",
|
||||
"--spec-draft-model",
|
||||
"-hfd",
|
||||
"-hfrd",
|
||||
"--spec-draft-hf",
|
||||
"--hf-repo-draft",
|
||||
"--spec-default",
|
||||
];
|
||||
const SPEC_USER_PREFIXES: &[&str] = &["--spec-ngram-", "--spec_ngram_"];
|
||||
|
||||
/// `specSidecarArch`.
|
||||
pub fn spec_sidecar_arch(arch: &str) -> bool {
|
||||
matches!(trim(arch).to_lowercase().as_str(), "eagle3" | "dflash")
|
||||
}
|
||||
|
||||
/// `specMode` : "off", "auto", "mtp", "ngram", "mtp+ngram", ou "" illisible.
|
||||
pub fn spec_mode(cfg: &Cfg) -> &'static str {
|
||||
match trim(get(cfg, "SPEC")).to_lowercase().as_str() {
|
||||
"" | "off" | "non" | "no" | "0" => "off",
|
||||
"auto" => "auto",
|
||||
"mtp" => "mtp",
|
||||
"ngram" => "ngram",
|
||||
"mtp+ngram" | "ngram+mtp" => "mtp+ngram",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// `specUserSet`.
|
||||
pub fn spec_user_set(extra: &[String], arg_env: &HashMap<String, String>) -> bool {
|
||||
if has_any_flag(extra, SPEC_USER_FLAGS)
|
||||
|| !has_flag_prefix(extra, SPEC_USER_PREFIXES).is_empty()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
[
|
||||
"LLAMA_ARG_SPEC_TYPE",
|
||||
"LLAMA_ARG_SPEC_DRAFT_MODEL",
|
||||
"LLAMA_ARG_SPEC_DRAFT_HF_REPO",
|
||||
]
|
||||
.iter()
|
||||
.any(|k| arg_env.get(*k).is_some_and(|v| !v.is_empty()))
|
||||
}
|
||||
|
||||
/// `helpSupportsMTP`.
|
||||
pub fn help_supports_mtp(help: &str) -> bool {
|
||||
help.contains("draft-mtp") && help.contains("--spec-draft-n-max")
|
||||
}
|
||||
|
||||
/// `ctxFixed` : (contexte vu par le moteur, chiffré > 0 ?).
|
||||
pub fn ctx_fixed(cfg: &Cfg, extra: &[String]) -> (String, bool) {
|
||||
let mut c = flag_value(extra, &["-c", "--ctx-size"]);
|
||||
if c.is_empty() {
|
||||
c = trim(get(cfg, "CTX")).to_string();
|
||||
}
|
||||
if c.is_empty() {
|
||||
c = "32768".into();
|
||||
}
|
||||
let ok = atoi(&c).is_some_and(|n| n > 0);
|
||||
(c, ok)
|
||||
}
|
||||
|
||||
/// `statefulSampler`.
|
||||
pub fn stateful_sampler(extra: &[String]) -> String {
|
||||
for v in flag_values(extra, &["--mirostat"]) {
|
||||
let v = trim(&v);
|
||||
if !v.is_empty() && v != "0" {
|
||||
return format!("--mirostat {v}");
|
||||
}
|
||||
}
|
||||
for v in flag_values(extra, &["--samplers"]) {
|
||||
if v.replace('-', "_").to_lowercase().contains("adaptive_p") {
|
||||
return "--samplers adaptive-p".into();
|
||||
}
|
||||
}
|
||||
for v in flag_values(extra, &["--sampling-seq", "--sampler-seq"]) {
|
||||
if v.contains('a') {
|
||||
return "--sampling-seq avec adaptive-p".into();
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `specAutoBlocker`.
|
||||
pub fn spec_auto_blocker(cfg: &Cfg, extra: &[String], si: &SysInfo) -> String {
|
||||
if si
|
||||
.gguf
|
||||
.as_ref()
|
||||
.is_some_and(|g| g.arch.eq_ignore_ascii_case("qwen4exp"))
|
||||
{
|
||||
return "qwen4exp : MTP trop récent pour ce moteur, SPEC=mtp pour l'imposer".into();
|
||||
}
|
||||
let why = fit_blocker(cfg, extra, si);
|
||||
if !why.is_empty() {
|
||||
return format!("placement manuel ({why}) — --fit ne compte pas la VRAM du brouillon");
|
||||
}
|
||||
if has_any_flag(extra, &["-dev", "--device"]) || !si.env("LLAMA_ARG_DEVICE").is_empty() {
|
||||
return "placement manuel (--device) — --fit ne compte pas la VRAM du brouillon".into();
|
||||
}
|
||||
if !ctx_fixed(cfg, extra).1 {
|
||||
return "contexte non chiffré — fit pourrait le réduire pour loger le brouillon".into();
|
||||
}
|
||||
if !si.mmproj.is_empty() || has_any_flag(extra, &["--mmproj", "-mm"]) {
|
||||
return "projecteur vision chargé".into();
|
||||
}
|
||||
if si.engine_build < SPEC_AUTO_MIN_BUILD {
|
||||
if si.engine_build <= 0 {
|
||||
return "build du moteur inconnu (compilé ou fork)".into();
|
||||
}
|
||||
return format!(
|
||||
"moteur b{}, b{SPEC_AUTO_MIN_BUILD} au moins",
|
||||
si.engine_build
|
||||
);
|
||||
}
|
||||
si.spec_auto_blocked.clone()
|
||||
}
|
||||
|
||||
/// Drapeaux, notes, et « vient de SPEC=auto ».
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SpecOut {
|
||||
pub args: Vec<String>,
|
||||
pub notes: Vec<String>,
|
||||
pub auto: bool,
|
||||
}
|
||||
|
||||
fn base_name(p: &str) -> String {
|
||||
crate::paths::base_name(p)
|
||||
}
|
||||
|
||||
/// `specArgs`.
|
||||
pub fn spec_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> SpecOut {
|
||||
if si.draft_is_model && !trim(get(cfg, "MODEL_DRAFT")).is_empty() {
|
||||
let mut c = cfg.clone();
|
||||
c.remove("MODEL_DRAFT");
|
||||
let mut out = spec_args(&c, extra, si);
|
||||
out.notes.insert(0, "MODEL_DRAFT est le modèle principal lui-même : ignoré — la tête MTP intégrée sert de brouillon sans recharger les poids (retire MODEL_DRAFT du preset)".into());
|
||||
return out;
|
||||
}
|
||||
let mode = spec_mode(cfg);
|
||||
let draft_key = trim(get(cfg, "MODEL_DRAFT"));
|
||||
match mode {
|
||||
"" => {
|
||||
return SpecOut {
|
||||
notes: vec![format!(
|
||||
"SPEC={} illisible (off, auto, mtp, ngram ou mtp+ngram) : sans décodage spéculatif",
|
||||
get(cfg, "SPEC")
|
||||
)],
|
||||
..SpecOut::default()
|
||||
};
|
||||
}
|
||||
"off" => {
|
||||
let mut out = SpecOut::default();
|
||||
if !draft_key.is_empty() {
|
||||
out.notes
|
||||
.push("MODEL_DRAFT ignoré : SPEC=off (auto ou mtp pour s'en servir)".into());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if spec_user_set(extra, &si.arg_env) {
|
||||
return SpecOut {
|
||||
notes: vec![
|
||||
"décodage spéculatif déjà réglé dans EXTRA_ARGS ou LLAMA_ARG_* : SPEC ignoré"
|
||||
.into(),
|
||||
],
|
||||
..SpecOut::default()
|
||||
};
|
||||
}
|
||||
let skip = |why: &str| SpecOut {
|
||||
notes: vec![format!("SPEC={mode} : {why} — sans décodage spéculatif")],
|
||||
..SpecOut::default()
|
||||
};
|
||||
|
||||
if mode == "ngram" || mode == "mtp+ngram" {
|
||||
let why = ngram_blocker(cfg, extra, si, mode);
|
||||
if !why.is_empty() {
|
||||
return skip(&why);
|
||||
}
|
||||
}
|
||||
if mode == "ngram" {
|
||||
let (args, notes) = ngram_spec_args(cfg, extra, si, "");
|
||||
return SpecOut {
|
||||
args,
|
||||
notes,
|
||||
auto: false,
|
||||
};
|
||||
}
|
||||
|
||||
let (src, mtp, why, quiet) = spec_source(mode, draft_key, si);
|
||||
if mode == "mtp+ngram" && !mtp {
|
||||
let why = if why.is_empty() {
|
||||
"MODEL_DRAFT n'est pas une tête MTP".to_string()
|
||||
} else {
|
||||
why
|
||||
};
|
||||
let (args, notes) = ngram_spec_args(
|
||||
cfg,
|
||||
extra,
|
||||
si,
|
||||
&format!("SPEC=mtp+ngram : sans MTP ({why}) → n-grammes seuls. "),
|
||||
);
|
||||
return SpecOut {
|
||||
args,
|
||||
notes,
|
||||
auto: false,
|
||||
};
|
||||
}
|
||||
if !why.is_empty() {
|
||||
return skip(&why);
|
||||
}
|
||||
if quiet {
|
||||
return SpecOut::default();
|
||||
}
|
||||
let mut args = src;
|
||||
if mode == "mtp+ngram" {
|
||||
if let Some(last) = args.last_mut() {
|
||||
*last = "draft-mtp,ngram-mod".into();
|
||||
}
|
||||
args.extend(ngram_mod_params());
|
||||
}
|
||||
let mut notes: Vec<String> = Vec::new();
|
||||
let mut auto = false;
|
||||
let label = if mtp {
|
||||
"tête MTP".to_string()
|
||||
} else {
|
||||
format!("brouillon {}", base_name(&si.draft))
|
||||
};
|
||||
if mode == "auto" {
|
||||
let why = spec_auto_blocker(cfg, extra, si);
|
||||
if !why.is_empty() {
|
||||
return skip(&why);
|
||||
}
|
||||
auto = true;
|
||||
notes.push(format!("SPEC=auto → {label} : sortie inchangée (chaque jeton est vérifié par le modèle), ~1-2 Go de VRAM en plus. Mesure le prefill ; SPEC=off pour couper."));
|
||||
} else {
|
||||
let mut note =
|
||||
format!("SPEC={mode} → {label} imposé : sortie inchangée, ~1-2 Go de VRAM en plus");
|
||||
if mode == "mtp+ngram" {
|
||||
note.push_str(
|
||||
" ; n-grammes en plus (ngram-mod, prioritaires quand ils trouvent une répétition)",
|
||||
);
|
||||
}
|
||||
let mut why = fit_blocker(cfg, extra, si);
|
||||
if !why.is_empty() || has_any_flag(extra, &["-dev", "--device"]) {
|
||||
if why.is_empty() {
|
||||
why = "--device".into();
|
||||
}
|
||||
note.push_str(&format!(" ; placement manuel ({why}) : --fit ne compte pas le brouillon, place-le avec -devd / -ngld si la VRAM manque"));
|
||||
}
|
||||
if !ctx_fixed(cfg, extra).1 {
|
||||
note.push_str(
|
||||
" ; contexte non chiffré : fit pourrait le réduire pour loger le brouillon",
|
||||
);
|
||||
}
|
||||
if si
|
||||
.gguf
|
||||
.as_ref()
|
||||
.is_some_and(|g| g.arch.eq_ignore_ascii_case("qwen4exp"))
|
||||
{
|
||||
note.push_str(" ; MTP qwen4exp très récent");
|
||||
}
|
||||
notes.push(note);
|
||||
if mode == "mtp+ngram" {
|
||||
notes.extend(ngram_cost_notes(cfg, extra, si));
|
||||
}
|
||||
}
|
||||
|
||||
let v = trim(get(cfg, "SPEC_N_MAX"));
|
||||
if !v.is_empty() {
|
||||
match atoi(v) {
|
||||
Some(n) if (1..=SPEC_N_MAX_LIMIT).contains(&n) => {
|
||||
if !si.help.contains("--spec-draft-n-max") {
|
||||
notes.push(
|
||||
"ce moteur ne connaît pas --spec-draft-n-max : SPEC_N_MAX ignoré".into(),
|
||||
);
|
||||
} else if has_any_flag(extra, &["--spec-draft-n-max"])
|
||||
|| !si.env("LLAMA_ARG_SPEC_DRAFT_N_MAX").is_empty()
|
||||
{
|
||||
} else {
|
||||
args.push("--spec-draft-n-max".into());
|
||||
args.push(n.to_string());
|
||||
}
|
||||
}
|
||||
_ => notes.push(format!(
|
||||
"SPEC_N_MAX={v} illisible (1 à 64 jetons) : défaut du moteur"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
let mut sampling = "greedy".to_string();
|
||||
let sv = trim(get(cfg, "SPEC_SAMPLING")).to_lowercase();
|
||||
match sv.as_str() {
|
||||
"" | "greedy" => {}
|
||||
"probabilistic" => {
|
||||
let st = stateful_sampler(extra);
|
||||
if mode == "auto" {
|
||||
notes.push("SPEC_SAMPLING=probabilistic ignoré avec SPEC=auto (SPEC=mtp pour le choisir) : greedy".into());
|
||||
} else if !st.is_empty() {
|
||||
notes.push(format!("SPEC_SAMPLING=probabilistic refusé avec {st} (cet échantillonneur avancerait sur des jetons rejetés) : greedy"));
|
||||
} else if !si.help.contains("probabilistic") {
|
||||
notes.push(
|
||||
"ce moteur ne connaît pas --spec-draft-sampling probabilistic : greedy".into(),
|
||||
);
|
||||
} else {
|
||||
sampling = sv.clone();
|
||||
}
|
||||
}
|
||||
_ => notes.push(format!(
|
||||
"SPEC_SAMPLING={sv} illisible (greedy ou probabilistic) : greedy"
|
||||
)),
|
||||
}
|
||||
if si.help.contains("--spec-draft-sampling")
|
||||
&& !has_any_flag(extra, &["--spec-draft-sampling"])
|
||||
&& si.env("LLAMA_ARG_SPEC_DRAFT_SAMPLING").is_empty()
|
||||
{
|
||||
args.push("--spec-draft-sampling".into());
|
||||
args.push(sampling);
|
||||
}
|
||||
SpecOut { args, notes, auto }
|
||||
}
|
||||
|
||||
/// `specSource` : (drapeaux, tête MTP ?, pourquoi aucun, rien à dire).
|
||||
pub fn spec_source(mode: &str, draft_key: &str, si: &SysInfo) -> (Vec<String>, bool, String, bool) {
|
||||
let none = |w: String| (Vec::new(), false, w, false);
|
||||
if !draft_key.is_empty() {
|
||||
if si.draft.is_empty() {
|
||||
let mut why = format!("MODEL_DRAFT={draft_key} introuvable");
|
||||
if !si.draft_err.is_empty() {
|
||||
why.push_str(&format!(" ({})", si.draft_err));
|
||||
}
|
||||
return none(why);
|
||||
}
|
||||
let Some(dg) = si.draft_gguf.as_ref() else {
|
||||
return none(
|
||||
"MODEL_DRAFT illisible (GGUF incomplet ou en cours de téléchargement ?)".into(),
|
||||
);
|
||||
};
|
||||
if dg.has_next_n_tensor {
|
||||
if !help_supports_mtp(&si.help) {
|
||||
return none("ce moteur ne connaît pas draft-mtp".into());
|
||||
}
|
||||
return (
|
||||
vec![
|
||||
"-md".into(),
|
||||
si.draft.clone(),
|
||||
"--spec-type".into(),
|
||||
"draft-mtp".into(),
|
||||
],
|
||||
true,
|
||||
String::new(),
|
||||
false,
|
||||
);
|
||||
}
|
||||
if spec_sidecar_arch(&dg.arch) {
|
||||
return none(format!(
|
||||
"MODEL_DRAFT est une tête {}, pas un modèle brouillon : règle -md et --spec-type dans EXTRA_ARGS",
|
||||
dg.arch
|
||||
));
|
||||
}
|
||||
if mode == "mtp+ngram" {
|
||||
return none(String::new());
|
||||
}
|
||||
if !si.help.contains("--spec-type") || !si.help.contains("draft-simple") {
|
||||
return none("ce moteur ne connaît pas --spec-type draft-simple".into());
|
||||
}
|
||||
return (
|
||||
vec![
|
||||
"-md".into(),
|
||||
si.draft.clone(),
|
||||
"--spec-type".into(),
|
||||
"draft-simple".into(),
|
||||
],
|
||||
false,
|
||||
String::new(),
|
||||
false,
|
||||
);
|
||||
}
|
||||
let Some(g) = si.gguf.as_ref() else {
|
||||
return none("métadonnées du modèle illisibles".into());
|
||||
};
|
||||
if !g.has_next_n_tensor {
|
||||
if mode != "auto" || g.next_n > 0 {
|
||||
return none(
|
||||
"pas de tête MTP dans ce fichier (publiée à part ? MODEL_DRAFT=mtp-….gguf)".into(),
|
||||
);
|
||||
}
|
||||
return (Vec::new(), false, String::new(), true);
|
||||
}
|
||||
if !help_supports_mtp(&si.help) {
|
||||
return none("ce moteur ne connaît pas draft-mtp".into());
|
||||
}
|
||||
(
|
||||
vec!["--spec-type".into(), "draft-mtp".into()],
|
||||
true,
|
||||
String::new(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// `ngramModParams`.
|
||||
pub fn ngram_mod_params() -> Vec<String> {
|
||||
vec![
|
||||
"--spec-ngram-mod-n-match".into(),
|
||||
SPEC_NGRAM_N_MATCH.to_string(),
|
||||
"--spec-ngram-mod-n-min".into(),
|
||||
SPEC_NGRAM_N_MIN.to_string(),
|
||||
"--spec-ngram-mod-n-max".into(),
|
||||
SPEC_NGRAM_N_MAX.to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
/// `ngramBlocker`.
|
||||
pub fn ngram_blocker(cfg: &Cfg, extra: &[String], si: &SysInfo, mode: &str) -> String {
|
||||
if !si.help.contains("--spec-ngram-mod-n-match") {
|
||||
return "ce moteur ne connaît pas --spec-ngram-mod-n-match".into();
|
||||
}
|
||||
if mode == "ngram" {
|
||||
let f = has_flag_prefix(extra, &["--spec-draft-", "--spec_draft_"]);
|
||||
if !f.is_empty() {
|
||||
return format!("{f} dans EXTRA_ARGS (brouillon réglé à la main)");
|
||||
}
|
||||
}
|
||||
let r = cpu_weights(cfg, extra, &si.arg_env, si.blocks());
|
||||
if !r.is_empty() {
|
||||
let thr = op_offload_threshold(cfg, si);
|
||||
if SPEC_NGRAM_N_MAX + 1 >= thr {
|
||||
return format!(
|
||||
"{r} : un lot de vérification de {} jetons atteint GGML_OP_OFFLOAD_MIN_BATCH ({thr}) et recopierait ces poids vers le GPU à chaque brouillon — OP_OFFLOAD_MIN_BATCH=128 pour l'essayer, à mesurer",
|
||||
SPEC_NGRAM_N_MAX + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// `opOffloadThreshold`.
|
||||
pub fn op_offload_threshold(cfg: &Cfg, si: &SysInfo) -> i64 {
|
||||
let mut v = trim(si.uenv("GGML_OP_OFFLOAD_MIN_BATCH")).to_string();
|
||||
if v.is_empty() {
|
||||
v = op_offload_min_batch_env(cfg, si).0;
|
||||
}
|
||||
match atoi(&v) {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => GGML_OFFLOAD_MIN_BATCH_DEFAULT,
|
||||
}
|
||||
}
|
||||
|
||||
/// `ngramSpecArgs`.
|
||||
pub fn ngram_spec_args(
|
||||
cfg: &Cfg,
|
||||
extra: &[String],
|
||||
si: &SysInfo,
|
||||
lead: &str,
|
||||
) -> (Vec<String>, Vec<String>) {
|
||||
let mut args = vec!["--spec-type".to_string(), "ngram-mod".to_string()];
|
||||
args.extend(ngram_mod_params());
|
||||
let mut notes = vec![format!(
|
||||
"{lead}SPEC=ngram → n-grammes du contexte (ngram-mod, brouillons de {SPEC_NGRAM_N_MIN} à {SPEC_NGRAM_N_MAX} jetons) : distribution inchangée (chaque jeton vérifié), ~16 Mo de RAM ; utile surtout en mode code, à mesurer (acceptation par nature dans /api/perf/summary)"
|
||||
)];
|
||||
if lead.is_empty() {
|
||||
if !trim(get(cfg, "MODEL_DRAFT")).is_empty() {
|
||||
notes.push("MODEL_DRAFT ignoré avec SPEC=ngram : SPEC=mtp+ngram pour les deux".into());
|
||||
} else if si.gguf.as_ref().is_some_and(|g| g.has_next_n_tensor) {
|
||||
notes.push(
|
||||
"tête MTP du modèle inutilisée avec SPEC=ngram : SPEC=mtp+ngram pour les deux"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
if !trim(get(cfg, "SPEC_N_MAX")).is_empty() || !trim(get(cfg, "SPEC_SAMPLING")).is_empty() {
|
||||
notes.push(
|
||||
"SPEC_N_MAX et SPEC_SAMPLING ignorés : les n-grammes n'ont pas de brouillon à régler"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
notes.extend(ngram_cost_notes(cfg, extra, si));
|
||||
(args, notes)
|
||||
}
|
||||
|
||||
/// `ngramCostNotes`.
|
||||
pub fn ngram_cost_notes(cfg: &Cfg, extra: &[String], si: &SysInfo) -> Vec<String> {
|
||||
let mut notes = Vec::new();
|
||||
if si.gguf.as_ref().is_some_and(|g| g.hybrid) {
|
||||
notes.push("modèle hybride : chaque brouillon n-gramme copie l'état récurrent (point de reprise en RAM hôte) et le recharge s'il est rejeté — compare le temps par tour, pas seulement le débit".into());
|
||||
}
|
||||
if !cpu_weights(cfg, extra, &si.arg_env, si.blocks()).is_empty()
|
||||
&& si.model_bytes > 0
|
||||
&& si.ram_mib > 0
|
||||
&& si.model_bytes > si.ram_mib << 20
|
||||
{
|
||||
notes.push("modèle plus gros que la RAM, poids sur CPU : un lot de vérification lit la plupart des experts de chaque couche — surveille les lectures disque, pas seulement les jetons/s".into());
|
||||
}
|
||||
if ngl_forced(cfg, extra) {
|
||||
notes.push(format!("NGL imposé (--fit inactif) : les logits de {} positions par slot prennent quelques dizaines de Mio de VRAM en plus", SPEC_NGRAM_N_MAX + 1));
|
||||
}
|
||||
notes
|
||||
}
|
||||
|
||||
// --- Jeton de tentative ------------------------------------------------------
|
||||
|
||||
pub const SPEC_ATTEMPT_KEY: &str = "spec_auto_attempt";
|
||||
pub const SPEC_FAILED_KEY: &str = "spec_auto_failed";
|
||||
pub const SPEC_FAILED_MAX: usize = 32;
|
||||
|
||||
/// `specAutoMark` : une combinaison preset × moteur.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct SpecAutoMark {
|
||||
#[serde(default)]
|
||||
pub fp: String,
|
||||
#[serde(default)]
|
||||
pub bin: String,
|
||||
#[serde(default)]
|
||||
pub build: i64,
|
||||
}
|
||||
|
||||
impl SpecAutoMark {
|
||||
pub fn key(&self) -> String {
|
||||
format!("{}|{}|{}", self.fp, self.bin, self.build)
|
||||
}
|
||||
}
|
||||
|
||||
/// `specAutoVerdict` : (raison de couper l'auto, inscrire l'échec ?).
|
||||
pub fn spec_auto_verdict(
|
||||
cur: &SpecAutoMark,
|
||||
attempt: Option<&SpecAutoMark>,
|
||||
failed: &HashMap<String, String>,
|
||||
) -> (String, bool) {
|
||||
if let Some(w) = failed.get(&cur.key()).filter(|w| !w.is_empty()) {
|
||||
return (w.clone(), false);
|
||||
}
|
||||
if attempt.is_some_and(|a| a.key() == cur.key()) {
|
||||
return (
|
||||
"le dernier lancement avec cette configuration n'a jamais répondu".into(),
|
||||
true,
|
||||
);
|
||||
}
|
||||
(String::new(), false)
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
//! Parallélisme de tenseurs (`backend_serve_split.go`, `SPLIT_MODE=tensor`,
|
||||
//! expérimental) : `-sm tensor` entre cartes CUDA, avec une longue liste de
|
||||
//! refus propres et `GGML_CUDA_ALLREDUCE=internal` (sommes dans le type
|
||||
//! natif, sans la compression BF16 de NCCL).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::{
|
||||
Cfg, SplitDev, SysInfo, atoi, ctx_fixed, effective_kv_types, env_truthy, flag_value, get,
|
||||
gguf_state_bytes, has_any_flag, is_external, kv_offloaded, side_slot_on, spec_mode,
|
||||
spec_user_set, tensor_override, trim,
|
||||
};
|
||||
|
||||
/// `splitTensorDenyArch`.
|
||||
pub fn split_tensor_deny_arch(arch: &str) -> bool {
|
||||
matches!(
|
||||
arch,
|
||||
"grok"
|
||||
| "mpt"
|
||||
| "plamo2"
|
||||
| "minicpm3"
|
||||
| "gemma3n"
|
||||
| "mamba"
|
||||
| "mamba2"
|
||||
| "jamba"
|
||||
| "falcon-h1"
|
||||
| "olmo2"
|
||||
| "olmoe"
|
||||
| "deepseek2"
|
||||
| "deepseek32"
|
||||
| "hy_v4"
|
||||
| "dots3note"
|
||||
| "glm-dsa"
|
||||
| "bitnet"
|
||||
| "t5"
|
||||
| "nemotron_h"
|
||||
| "nemotron_h_moe"
|
||||
| "granitehybrid"
|
||||
| "minimax-01"
|
||||
| "minimax-m2"
|
||||
| "minimax-m3"
|
||||
| "mistral4"
|
||||
| "kimi-linear"
|
||||
| "bailingmoe3"
|
||||
| "kimi-k3"
|
||||
| "glm5-next"
|
||||
| "qwen3tts"
|
||||
| "qwen4exp"
|
||||
)
|
||||
}
|
||||
|
||||
/// `splitArgEnv`.
|
||||
pub const SPLIT_ARG_ENV: &[&str] = &[
|
||||
"LLAMA_ARG_FLASH_ATTN",
|
||||
"LLAMA_ARG_BACKEND_SAMPLING",
|
||||
"LLAMA_ARG_TENSOR_SPLIT",
|
||||
];
|
||||
|
||||
/// `splitModeKey` : "" (rien à poser), "tensor", ou "?" si illisible.
|
||||
pub fn split_mode_key(cfg: &Cfg) -> &'static str {
|
||||
match trim(get(cfg, "SPLIT_MODE")).to_lowercase().as_str() {
|
||||
"" | "off" | "layer" | "non" | "no" | "0" => "",
|
||||
"tensor" => "tensor",
|
||||
_ => "?",
|
||||
}
|
||||
}
|
||||
|
||||
/// `helpSupportsSplitTensor`.
|
||||
pub fn help_supports_split_tensor(help: &str) -> bool {
|
||||
help.contains("{none,layer,row,tensor}")
|
||||
}
|
||||
|
||||
/// Décision de SPLIT_MODE=tensor (`splitPlan`).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SplitPlan {
|
||||
pub on: bool,
|
||||
pub why: String,
|
||||
pub devs: Vec<SplitDev>,
|
||||
pub need_mib: i64,
|
||||
}
|
||||
|
||||
fn refuse(why: impl Into<String>) -> SplitPlan {
|
||||
SplitPlan {
|
||||
why: why.into(),
|
||||
..SplitPlan::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn split_kv_ok(t: &str) -> bool {
|
||||
matches!(trim(t).to_lowercase().as_str(), "" | "f16" | "bf16" | "f32")
|
||||
}
|
||||
|
||||
fn kv_name(t: &str) -> &str {
|
||||
if t.is_empty() { "f16" } else { t }
|
||||
}
|
||||
|
||||
/// `splitPlan`.
|
||||
pub fn split_plan(cfg: &Cfg, extra: &[String], si: &SysInfo) -> SplitPlan {
|
||||
match split_mode_key(cfg) {
|
||||
"" => return SplitPlan::default(),
|
||||
"?" => {
|
||||
return refuse(format!(
|
||||
"SPLIT_MODE={} illisible (tensor ou off)",
|
||||
trim(get(cfg, "SPLIT_MODE"))
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if is_external(cfg) {
|
||||
return refuse("preset externe");
|
||||
}
|
||||
if !help_supports_split_tensor(&si.help) {
|
||||
return refuse("ce moteur ne propose pas -sm tensor");
|
||||
}
|
||||
let sm = flag_value(extra, &["-sm", "--split-mode"]);
|
||||
if !sm.is_empty() {
|
||||
return refuse(format!("-sm {sm} déjà dans EXTRA_ARGS"));
|
||||
}
|
||||
if !si.env("LLAMA_ARG_SPLIT_MODE").is_empty() {
|
||||
return refuse("LLAMA_ARG_SPLIT_MODE déjà posée");
|
||||
}
|
||||
let mut fa = flag_value(extra, &["-fa", "--flash-attn"]);
|
||||
if fa.is_empty() {
|
||||
fa = si.env("LLAMA_ARG_FLASH_ATTN").into();
|
||||
}
|
||||
if matches!(
|
||||
trim(&fa).to_lowercase().as_str(),
|
||||
"off" | "disabled" | "false" | "0"
|
||||
) {
|
||||
return refuse(format!(
|
||||
"Flash Attention coupée (-fa {fa}) : le mode tensor l'exige"
|
||||
));
|
||||
}
|
||||
if has_any_flag(extra, &["-bs", "--backend-sampling"])
|
||||
|| env_truthy(si.env("LLAMA_ARG_BACKEND_SAMPLING"))
|
||||
{
|
||||
return refuse(
|
||||
"échantillonnage sur GPU (--backend-sampling) non pris en charge en mode tensor",
|
||||
);
|
||||
}
|
||||
let (k, v, _) = effective_kv_types(cfg, extra, &si.arg_env);
|
||||
if !split_kv_ok(&k) || !split_kv_ok(&v) {
|
||||
return refuse(format!(
|
||||
"cache KV {}/{} : le mode tensor n'accepte que f16, bf16 ou f32 (jamais réécrit d'office : la VRAM changerait)",
|
||||
kv_name(&k),
|
||||
kv_name(&v)
|
||||
));
|
||||
}
|
||||
let r = tensor_override(extra, &si.arg_env);
|
||||
if !r.is_empty() {
|
||||
return refuse(format!("{r} : le mode tensor met tout sur les cartes"));
|
||||
}
|
||||
if !kv_offloaded(extra, &si.arg_env) {
|
||||
return refuse("cache KV sur CPU");
|
||||
}
|
||||
let g = match si.gguf.as_ref() {
|
||||
Some(g) if si.model_bytes > 0 => g,
|
||||
_ => return refuse("métadonnées ou taille du modèle illisibles"),
|
||||
};
|
||||
if split_tensor_deny_arch(&g.arch.to_lowercase()) {
|
||||
return refuse(format!("architecture {} exclue du mode tensor", g.arch));
|
||||
}
|
||||
let (_, why) = split_ngl(cfg, extra, g.block_count);
|
||||
if !why.is_empty() {
|
||||
return refuse(why);
|
||||
}
|
||||
let (ctx_s, ok) = ctx_fixed(cfg, extra);
|
||||
if !ok {
|
||||
return refuse(format!(
|
||||
"contexte « {ctx_s} » non chiffré : pas de --fit en mode tensor, donne un CTX explicite"
|
||||
));
|
||||
}
|
||||
if side_slot_on(cfg) {
|
||||
return refuse("SIDE_SLOT : cache doublé, sans --fit pour rattraper");
|
||||
}
|
||||
if spec_mode(cfg) != "off" || spec_user_set(extra, &si.arg_env) {
|
||||
return refuse(
|
||||
"décodage spéculatif : brouillon et vérification par lots non validés en mode tensor",
|
||||
);
|
||||
}
|
||||
let (devs, why) = split_devices(extra, si);
|
||||
if !why.is_empty() {
|
||||
return refuse(why);
|
||||
}
|
||||
let ctx = atoi(&ctx_s).unwrap_or(0);
|
||||
let (per_tok, rec) = gguf_state_bytes(g, &k, &v);
|
||||
if per_tok <= 0.0 {
|
||||
return refuse("taille du cache KV inconnue pour ce modèle");
|
||||
}
|
||||
let total: i64 = devs.iter().map(|d| d.total_mib).sum();
|
||||
let mm = if si.mmproj.is_empty() {
|
||||
0
|
||||
} else {
|
||||
std::fs::metadata(&si.mmproj).map_or(0, |m| m.len() as i64)
|
||||
};
|
||||
let mib = 1048576.0;
|
||||
let need = (si.model_bytes + mm) as f64 + per_tok * ctx as f64 + rec as f64;
|
||||
let need_mib = (need / mib).ceil() as i64;
|
||||
if need > 0.9 * total as f64 * mib {
|
||||
return refuse(format!(
|
||||
"VRAM insuffisante sans --fit : ~{need_mib} Mio (poids + cache de {ctx} jetons) pour {total} Mio sur {} cartes — CTX n'est jamais réduit d'office",
|
||||
devs.len()
|
||||
));
|
||||
}
|
||||
SplitPlan {
|
||||
on: true,
|
||||
why: String::new(),
|
||||
devs,
|
||||
need_mib,
|
||||
}
|
||||
}
|
||||
|
||||
/// `splitNGL`.
|
||||
pub fn split_ngl(cfg: &Cfg, extra: &[String], blocks: i64) -> (Vec<String>, String) {
|
||||
let all = |v: &str| -> bool {
|
||||
let v = trim(v).to_lowercase();
|
||||
if v == "all" || v == "999" {
|
||||
return true;
|
||||
}
|
||||
atoi(&v).is_some_and(|n| blocks > 0 && n > blocks)
|
||||
};
|
||||
let v = flag_value(extra, &["-ngl", "--n-gpu-layers", "--gpu-layers"]);
|
||||
if !v.is_empty() {
|
||||
if all(&v) {
|
||||
return (Vec::new(), String::new());
|
||||
}
|
||||
return (
|
||||
Vec::new(),
|
||||
format!(
|
||||
"-ngl {v} dans EXTRA_ARGS : le mode tensor veut toutes les couches sur les cartes (all)"
|
||||
),
|
||||
);
|
||||
}
|
||||
let v = trim(get(cfg, "NGL"));
|
||||
if v.is_empty() || v.eq_ignore_ascii_case("auto") || all(v) {
|
||||
(vec!["-ngl".into(), "all".into()], String::new())
|
||||
} else {
|
||||
(
|
||||
Vec::new(),
|
||||
format!("NGL={v} : le mode tensor veut toutes les couches sur les cartes (all)"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// `splitDevices`.
|
||||
pub fn split_devices(extra: &[String], si: &SysInfo) -> (Vec<SplitDev>, String) {
|
||||
if si.split_devs.is_empty() {
|
||||
return (
|
||||
Vec::new(),
|
||||
"cartes inconnues (--list-devices n'a rien répondu)".into(),
|
||||
);
|
||||
}
|
||||
let mut devs = si.split_devs.clone();
|
||||
let mut sel = flag_value(extra, &["-dev", "--device"]);
|
||||
if sel.is_empty() {
|
||||
sel = si.env("LLAMA_ARG_DEVICE").into();
|
||||
}
|
||||
let sel = trim(&sel);
|
||||
if !sel.is_empty() {
|
||||
devs = Vec::new();
|
||||
for name in sel.split(',') {
|
||||
let name = trim(name);
|
||||
match si
|
||||
.split_devs
|
||||
.iter()
|
||||
.find(|d| d.id.eq_ignore_ascii_case(name))
|
||||
{
|
||||
Some(d) => devs.push(d.clone()),
|
||||
None => {
|
||||
return (Vec::new(), format!("--device {name} inconnu de ce moteur"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for d in &devs {
|
||||
if !d.id.to_uppercase().starts_with("CUDA") {
|
||||
return (Vec::new(), format!("{} n'est pas une carte CUDA", d.id));
|
||||
}
|
||||
}
|
||||
if devs.len() < 2 {
|
||||
return (Vec::new(), "une seule carte".into());
|
||||
}
|
||||
(devs, String::new())
|
||||
}
|
||||
|
||||
/// `splitTS` : répartition au prorata de la VRAM totale, en pour cent.
|
||||
pub fn split_ts(devs: &[SplitDev]) -> String {
|
||||
let total: i64 = devs.iter().map(|d| d.total_mib).sum();
|
||||
if total <= 0 {
|
||||
return String::new();
|
||||
}
|
||||
devs.iter()
|
||||
.map(|d| go_round(100.0 * d.total_mib as f64 / total as f64).to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
/// `math.Round` : moitié arrondie loin de zéro (comme `f64::round`).
|
||||
fn go_round(x: f64) -> i64 {
|
||||
x.round() as i64
|
||||
}
|
||||
|
||||
/// Ce que SPLIT_MODE=tensor ajoute (`splitTensorArgs`).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SplitTensor {
|
||||
pub on: bool,
|
||||
pub args: Vec<String>,
|
||||
pub ngl: Vec<String>,
|
||||
pub env: BTreeMap<String, String>,
|
||||
pub notes: Vec<String>,
|
||||
}
|
||||
|
||||
/// `splitTensorArgs`.
|
||||
pub fn split_tensor_args(cfg: &Cfg, extra: &[String], si: &SysInfo) -> SplitTensor {
|
||||
let plan = split_plan(cfg, extra, si);
|
||||
let mut out = SplitTensor::default();
|
||||
if !plan.on {
|
||||
if !plan.why.is_empty() {
|
||||
out.notes.push(format!(
|
||||
"SPLIT_MODE=tensor refusé ({}) : découpe par couches, comme sans la clé",
|
||||
plan.why
|
||||
));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
let g = si.gguf.as_ref().expect("plan accepté : GGUF lu");
|
||||
out.on = true;
|
||||
out.ngl = split_ngl(cfg, extra, g.block_count).0;
|
||||
out.args = vec!["-sm".into(), "tensor".into()];
|
||||
let mut ts = String::new();
|
||||
if !has_any_flag(extra, &["-ts", "--tensor-split"])
|
||||
&& si.env("LLAMA_ARG_TENSOR_SPLIT").is_empty()
|
||||
{
|
||||
ts = split_ts(&plan.devs);
|
||||
if !ts.is_empty() {
|
||||
out.args.push("-ts".into());
|
||||
out.args.push(ts.clone());
|
||||
}
|
||||
}
|
||||
if !has_any_flag(extra, &["-fa", "--flash-attn"]) && si.env("LLAMA_ARG_FLASH_ATTN").is_empty() {
|
||||
out.args.push("-fa".into());
|
||||
out.args.push("on".into());
|
||||
}
|
||||
let mut note = format!(
|
||||
"SPLIT_MODE=tensor (expérimental) : -sm tensor sur {} cartes",
|
||||
plan.devs.len()
|
||||
);
|
||||
if !ts.is_empty() {
|
||||
note.push_str(&format!(", -ts {ts} (VRAM totale)"));
|
||||
}
|
||||
note.push_str(&format!(", ~{} Mio estimés sans --fit ; décodage plus rapide, prefill plus lent — compare le temps par tour", plan.need_mib));
|
||||
let ar = trim(si.uenv("GGML_CUDA_ALLREDUCE")).to_lowercase();
|
||||
match ar.as_str() {
|
||||
"" => {
|
||||
out.env.insert("GGML_CUDA_ALLREDUCE".into(), "internal".into());
|
||||
note.push_str(" ; GGML_CUDA_ALLREDUCE=internal : sommes dans le type natif (NCCL les compresserait en BF16, avec perte). « NCCL not compiled in » ou « NCCL is unavailable » au journal : sans effet ici. GeForce sans P2P : les sommes passent par l'hôte ; GGML_CUDA_P2P=1 à poser soi-même si le pilote le permet");
|
||||
}
|
||||
"nccl" => note.push_str(" ; avertissement : GGML_CUDA_ALLREDUCE=nccl (environnement) — NCCL compresse en BF16 les grosses réductions du prefill : calcul avec perte. « NCCL not compiled in » au journal = repli sur la réduction interne"),
|
||||
_ => note.push_str(&format!(" ; GGML_CUDA_ALLREDUCE={ar} (environnement) gardé")),
|
||||
}
|
||||
if g.hybrid {
|
||||
note.push_str(" ; modèle hybride : état récurrent et points de reprise répartis entre cartes, non validés en amont");
|
||||
}
|
||||
out.notes.push(note);
|
||||
out
|
||||
}
|
||||
|
||||
/// `parseListDevices` : les cartes de `llama-server --list-devices`.
|
||||
pub fn parse_list_devices(out: &str) -> Vec<SplitDev> {
|
||||
let re =
|
||||
regex::Regex::new(r"^\s*([A-Za-z]+\d+):\s*(.+?)\s*\((\d+)\s*MiB,\s*(\d+)\s*MiB free\)\s*$")
|
||||
.expect("motif");
|
||||
out.split('\n')
|
||||
.filter_map(|line| {
|
||||
let m = re.captures(line.trim_end_matches('\r'))?;
|
||||
Some(SplitDev {
|
||||
id: m[1].to_string(),
|
||||
total_mib: m[3].parse().unwrap_or(0),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
//! Threads CPU de llama-server (`backend_serve_threads.go`) : sans réglage,
|
||||
//! AUCUN drapeau — llama.cpp prend ses cœurs physiques. Un conteneur à
|
||||
//! l'étroit (cpuset, quota CFS) est sondé : on impose alors -t.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
use super::{atoi, flag_value, has_any_flag, trim};
|
||||
|
||||
/// Ce que la sonde de conteneur a conclu (`cpuBudget`). `n` = 0 : rien.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CpuBudget {
|
||||
pub n: i64,
|
||||
pub engine: i64,
|
||||
pub why: String,
|
||||
}
|
||||
|
||||
/// `threadCount` : vide ou 0 = auto ; illisible ou négatif ignoré, en le
|
||||
/// disant.
|
||||
pub fn thread_count(key: &str, v: &str) -> (i64, String) {
|
||||
let v = trim(v);
|
||||
if v.is_empty() {
|
||||
return (0, String::new());
|
||||
}
|
||||
match atoi(v) {
|
||||
Some(n) if n >= 0 => (n, String::new()),
|
||||
_ => (
|
||||
0,
|
||||
format!("{key}={v} ignoré (pas un nombre de threads ≥ 0) → automatique"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// `batchCPUFlags`.
|
||||
pub const BATCH_CPU_FLAGS: &[&str] = &[
|
||||
"-Cb",
|
||||
"--cpu-mask-batch",
|
||||
"-Crb",
|
||||
"--cpu-range-batch",
|
||||
"--cpu-strict-batch",
|
||||
"--prio-batch",
|
||||
"--poll-batch",
|
||||
];
|
||||
|
||||
/// `threadArgs` : -t / -tb.
|
||||
pub fn thread_args(
|
||||
threads: &str,
|
||||
threads_batch: &str,
|
||||
extra: &[String],
|
||||
auto: &CpuBudget,
|
||||
) -> (Vec<String>, Vec<String>) {
|
||||
let mut args = Vec::new();
|
||||
let mut notes = Vec::new();
|
||||
let (t, note) = thread_count("THREADS", threads);
|
||||
if !note.is_empty() {
|
||||
notes.push(note);
|
||||
}
|
||||
let mut t_val = String::new();
|
||||
if has_any_flag(extra, &["-t", "--threads"]) {
|
||||
t_val = flag_value(extra, &["-t", "--threads"]);
|
||||
} else if t > 0 {
|
||||
t_val = t.to_string();
|
||||
args.push("-t".into());
|
||||
args.push(t_val.clone());
|
||||
} else if auto.n > 0
|
||||
&& !has_any_flag(
|
||||
extra,
|
||||
&["-C", "--cpu-mask", "-Cr", "--cpu-range", "--cpu-strict"],
|
||||
)
|
||||
{
|
||||
t_val = auto.n.to_string();
|
||||
args.push("-t".into());
|
||||
args.push(t_val.clone());
|
||||
notes.push(format!(
|
||||
"threads auto → -t {} : {} (llama.cpp seul en lancerait {})",
|
||||
auto.n, auto.why, auto.engine
|
||||
));
|
||||
}
|
||||
let (tb, note) = thread_count("THREADS_BATCH", threads_batch);
|
||||
if !note.is_empty() {
|
||||
notes.push(note);
|
||||
}
|
||||
if has_any_flag(extra, &["-tb", "--threads-batch"]) {
|
||||
} else if tb > 0 {
|
||||
args.push("-tb".into());
|
||||
args.push(tb.to_string());
|
||||
} else if has_any_flag(extra, BATCH_CPU_FLAGS) {
|
||||
if t_val.is_empty() {
|
||||
t_val = "0".into();
|
||||
notes.push("EXTRA_ARGS règle le CPU du batch (-Cb, --poll-batch…) : -tb 0 posé pour que llama.cpp le garde (tous les threads logiques) ; THREADS_BATCH pour choisir le nombre".into());
|
||||
}
|
||||
args.push("-tb".into());
|
||||
args.push(t_val);
|
||||
}
|
||||
(args, notes)
|
||||
}
|
||||
|
||||
fn read(root: &Path, rel: &str) -> Option<String> {
|
||||
std::fs::read(root.join(rel))
|
||||
.ok()
|
||||
.map(|b| String::from_utf8_lossy(&b).into_owned())
|
||||
}
|
||||
|
||||
/// `cpusetThreads` : sonde de conteneur (Linux) ; `root` = la racine « / ».
|
||||
pub fn cpuset_threads(root: &Path) -> CpuBudget {
|
||||
let Some(online) = read_cpu_list(root, "sys/devices/system/cpu/online") else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
if online.is_empty() {
|
||||
return CpuBudget::default();
|
||||
}
|
||||
let Some(allowed) = proc_allowed_cpus(root) else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
let allowed = intersect_cpus(&allowed, &online);
|
||||
if allowed.is_empty() {
|
||||
return CpuBudget::default();
|
||||
}
|
||||
let restricted = allowed.len() < online.len();
|
||||
let quota = cgroup_cpu_quota(root);
|
||||
if !restricted && quota == 0 {
|
||||
return CpuBudget::default();
|
||||
}
|
||||
let atom = read_cpu_list(root, "sys/devices/cpu_atom/cpus").unwrap_or_default();
|
||||
let Some(host_all) = physical_cores(root, &online, &[]) else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
let Some(mut host_math) = physical_cores(root, &online, &atom) else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
if host_math == 0 {
|
||||
host_math = host_all;
|
||||
}
|
||||
let (mut n, engine, mut why);
|
||||
if restricted {
|
||||
engine = host_all;
|
||||
let Some(x) = physical_cores(root, &allowed, &atom) else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
n = x;
|
||||
if n == 0 {
|
||||
let Some(x) = physical_cores(root, &allowed, &[]) else {
|
||||
return CpuBudget::default();
|
||||
};
|
||||
n = x;
|
||||
}
|
||||
why = format!("cpuset du conteneur, {n} cœurs physiques permis sur {host_all}");
|
||||
if !atom.is_empty() {
|
||||
why.push_str(" (cœurs E écartés)");
|
||||
}
|
||||
} else {
|
||||
engine = host_math;
|
||||
n = host_math;
|
||||
why = String::new();
|
||||
}
|
||||
if quota > 0 && quota < n {
|
||||
n = quota;
|
||||
why = format!("quota cgroup de {quota} CPU");
|
||||
}
|
||||
if n <= 0 || n >= engine {
|
||||
return CpuBudget::default();
|
||||
}
|
||||
CpuBudget { n, engine, why }
|
||||
}
|
||||
|
||||
/// `physicalCores` : une chaîne `thread_siblings` distincte par cœur.
|
||||
pub fn physical_cores(root: &Path, cpus: &[i64], exclude: &[i64]) -> Option<i64> {
|
||||
let mut seen = BTreeSet::new();
|
||||
for c in cpus {
|
||||
if exclude.contains(c) {
|
||||
continue;
|
||||
}
|
||||
let b = read(
|
||||
root,
|
||||
&format!("sys/devices/system/cpu/cpu{c}/topology/thread_siblings"),
|
||||
)?;
|
||||
seen.insert(trim(&b).to_string());
|
||||
}
|
||||
Some(seen.len() as i64)
|
||||
}
|
||||
|
||||
/// `procAllowedCPUs` : `Cpus_allowed_list` de /proc/self/status.
|
||||
fn proc_allowed_cpus(root: &Path) -> Option<Vec<i64>> {
|
||||
let b = read(root, "proc/self/status")?;
|
||||
for line in b.split('\n') {
|
||||
if let Some(v) = line.strip_prefix("Cpus_allowed_list:") {
|
||||
return parse_cpu_list(v);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `cgroupCPUQuota` : quota arrondi au CPU supérieur, 0 sans quota.
|
||||
pub fn cgroup_cpu_quota(root: &Path) -> i64 {
|
||||
if let Some(b) = read(root, "sys/fs/cgroup/cpu.max") {
|
||||
let f: Vec<&str> = b.split_whitespace().collect();
|
||||
if f.is_empty() || f[0] == "max" {
|
||||
return 0;
|
||||
}
|
||||
let period = f.get(1).copied().unwrap_or("100000");
|
||||
return ceil_quota(f[0], period);
|
||||
}
|
||||
for dir in ["sys/fs/cgroup/cpu", "sys/fs/cgroup/cpu,cpuacct"] {
|
||||
let Some(q) = read(root, &format!("{dir}/cpu.cfs_quota_us")) else {
|
||||
continue;
|
||||
};
|
||||
let Some(p) = read(root, &format!("{dir}/cpu.cfs_period_us")) else {
|
||||
return 0;
|
||||
};
|
||||
return ceil_quota(trim(&q), trim(&p));
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
fn ceil_quota(quota: &str, period: &str) -> i64 {
|
||||
match (atoi(quota), atoi(period)) {
|
||||
(Some(q), Some(p)) if q > 0 && p > 0 => ((q + p - 1) / p).max(1),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_cpu_list(root: &Path, rel: &str) -> Option<Vec<i64>> {
|
||||
parse_cpu_list(&read(root, rel)?)
|
||||
}
|
||||
|
||||
/// `parseCPUList` : « 0-3,8-11 » → liste triée sans doublon ; syntaxe
|
||||
/// inconnue → `None`.
|
||||
pub fn parse_cpu_list(s: &str) -> Option<Vec<i64>> {
|
||||
let mut set = BTreeSet::new();
|
||||
for part in trim(s).split(',') {
|
||||
let part = trim(part);
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (lo, hi) = match part.split_once('-') {
|
||||
Some((l, h)) => (l, Some(h)),
|
||||
None => (part, None),
|
||||
};
|
||||
let a = atoi(lo)?;
|
||||
let b = match hi {
|
||||
Some(h) => atoi(h)?,
|
||||
None => a,
|
||||
};
|
||||
if a < 0 || b < a || b - a > 1 << 16 {
|
||||
return None;
|
||||
}
|
||||
for c in a..=b {
|
||||
set.insert(c);
|
||||
}
|
||||
}
|
||||
Some(set.into_iter().collect())
|
||||
}
|
||||
|
||||
fn intersect_cpus(a: &[i64], b: &[i64]) -> Vec<i64> {
|
||||
a.iter().copied().filter(|c| b.contains(c)).collect()
|
||||
}
|
||||
@@ -30,6 +30,7 @@ fn faux_go(dir: &Path, corps: &str) -> PathBuf {
|
||||
|
||||
fn opts(go: PathBuf) -> Opts {
|
||||
Opts {
|
||||
serve: (go.clone(), vec!["serve".into()]),
|
||||
go_bin: go,
|
||||
check_delay: Duration::from_millis(500),
|
||||
grace: Duration::from_secs(2),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,523 @@
|
||||
//! Lot 14 : `loki serve` composé en Rust (`LOKI_SERVE=rust`).
|
||||
//!
|
||||
//! Le journal tiré du Go (`internal/loki/parity_serve_test.go` →
|
||||
//! `tests/parity_serve/serve.json`) est rejoué : ligne de commande,
|
||||
//! environnement et notes de ~1 900 lancements (machines × presets × aides de
|
||||
//! moteur × variables), puis les sondes pures.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use loki_rs::gguf::GgufMeta;
|
||||
use loki_rs::serve::{self, CpuBudget, SplitDev, SysInfo};
|
||||
use serde_json::Value;
|
||||
|
||||
fn fixture() -> Value {
|
||||
serde_json::from_str(include_str!("parity_serve/serve.json")).expect("journal")
|
||||
}
|
||||
|
||||
fn i(v: &Value, k: &str) -> i64 {
|
||||
v.get(k).and_then(Value::as_i64).unwrap_or(0)
|
||||
}
|
||||
fn st(v: &Value, k: &str) -> String {
|
||||
v.get(k).and_then(Value::as_str).unwrap_or("").to_string()
|
||||
}
|
||||
fn b(v: &Value, k: &str) -> bool {
|
||||
v.get(k).and_then(Value::as_bool).unwrap_or(false)
|
||||
}
|
||||
|
||||
fn gguf_from(v: &Value) -> Option<GgufMeta> {
|
||||
if v.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(GgufMeta {
|
||||
version: i(v, "Version"),
|
||||
arch: st(v, "Arch"),
|
||||
block_count: i(v, "BlockCount"),
|
||||
next_n: i(v, "NextN"),
|
||||
context_length: i(v, "ContextLength"),
|
||||
expert_count: i(v, "ExpertCount"),
|
||||
key_len: i(v, "KeyLen"),
|
||||
val_len: i(v, "ValLen"),
|
||||
head_count_kv: i(v, "HeadCountKV"),
|
||||
head_count_kv_layers: v
|
||||
.get("HeadCountKVLayers")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().map(|x| x.as_i64().unwrap()).collect()),
|
||||
full_attn_interval: i(v, "FullAttnInterval"),
|
||||
hybrid: b(v, "Hybrid"),
|
||||
embedding_length: i(v, "EmbeddingLength"),
|
||||
head_count: i(v, "HeadCount"),
|
||||
ssm_conv: i(v, "SSMConv"),
|
||||
ssm_inner: i(v, "SSMInner"),
|
||||
ssm_state: i(v, "SSMState"),
|
||||
ssm_groups: i(v, "SSMGroups"),
|
||||
has_next_n_tensor: b(v, "HasNextNTensor"),
|
||||
tensor_count: i(v, "TensorCount"),
|
||||
})
|
||||
}
|
||||
|
||||
fn map_from(v: &Value) -> HashMap<String, String> {
|
||||
v.as_object()
|
||||
.map(|o| {
|
||||
o.iter()
|
||||
.map(|(k, x)| (k.clone(), x.as_str().unwrap().to_string()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn si_from(m: &Value) -> SysInfo {
|
||||
let cpu = &m["CPU"];
|
||||
SysInfo {
|
||||
help: st(m, "Help"),
|
||||
model: st(m, "Model"),
|
||||
mmproj: st(m, "MMProj"),
|
||||
api_key: st(m, "APIKey"),
|
||||
cpu: CpuBudget {
|
||||
n: i(cpu, "N"),
|
||||
engine: i(cpu, "Engine"),
|
||||
why: st(cpu, "Why"),
|
||||
},
|
||||
launch_queues: st(m, "LaunchQueues"),
|
||||
gpus: i(m, "GPUs"),
|
||||
arg_env: map_from(&m["ArgEnv"]),
|
||||
user_env: map_from(&m["UserEnv"]),
|
||||
gguf: gguf_from(&m["GGUF"]),
|
||||
model_bytes: i(m, "ModelBytes"),
|
||||
ram_mib: i(m, "RAMMiB"),
|
||||
ram_avail_mib: i(m, "RAMAvailMiB"),
|
||||
vram_mib: i(m, "VRAMMiB"),
|
||||
unified_mem: b(m, "UnifiedMem"),
|
||||
engine_vram_mib: i(m, "EngineVRAMMiB"),
|
||||
slot_dir: st(m, "SlotDir"),
|
||||
engine_build: i(m, "EngineBuild"),
|
||||
draft: st(m, "Draft"),
|
||||
draft_err: st(m, "DraftErr"),
|
||||
draft_gguf: gguf_from(&m["DraftGGUF"]),
|
||||
draft_is_model: b(m, "DraftIsModel"),
|
||||
spec_auto_blocked: st(m, "SpecAutoBlocked"),
|
||||
split_devs: m["SplitDevs"]
|
||||
.as_array()
|
||||
.map(|a| {
|
||||
a.iter()
|
||||
.map(|d| SplitDev {
|
||||
id: st(d, "ID"),
|
||||
total_mib: i(d, "TotalMiB"),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn strs(v: &Value) -> Vec<String> {
|
||||
v.as_array()
|
||||
.map(|a| a.iter().map(|x| x.as_str().unwrap().to_string()).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parite_des_lignes_de_commande() {
|
||||
let fx = fixture();
|
||||
let machines = fx["machines"].as_array().unwrap();
|
||||
let presets = fx["presets"].as_array().unwrap();
|
||||
let helps = fx["helps"].as_array().unwrap();
|
||||
let arg_envs = fx["arg_envs"].as_array().unwrap();
|
||||
let user_envs = fx["user_envs"].as_array().unwrap();
|
||||
let mut n = 0;
|
||||
for c in fx["cases"].as_array().unwrap() {
|
||||
let (m, p, h, e) = (
|
||||
i(c, "m") as usize,
|
||||
i(c, "c") as usize,
|
||||
i(c, "h") as usize,
|
||||
i(c, "e") as usize,
|
||||
);
|
||||
let mut si = si_from(&machines[m]);
|
||||
si.help = helps[h].as_str().unwrap().to_string();
|
||||
si.arg_env = map_from(&arg_envs[e]);
|
||||
si.user_env = if (m + p) % 3 == 1 {
|
||||
map_from(&user_envs[1])
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let cfg = map_from(&presets[p]);
|
||||
let extra = loki_rs::models::split_args(cfg.get("EXTRA_ARGS").map_or("", String::as_str));
|
||||
let got = serve::build_serve_args(&cfg, &extra, "/app/llama-server", &si);
|
||||
let ctx = format!(
|
||||
"machine {m}, preset {p} {:?}, aide {h}, variables {e}",
|
||||
presets[p]
|
||||
);
|
||||
assert_eq!(got.args, strs(&c["args"]), "{ctx} : arguments");
|
||||
let want_env: BTreeMap<String, String> = map_from(&c["env"]).into_iter().collect();
|
||||
assert_eq!(got.env, want_env, "{ctx} : environnement");
|
||||
assert_eq!(got.notes, strs(&c["notes"]), "{ctx} : notes");
|
||||
n += 1;
|
||||
}
|
||||
assert!(n > 1500, "{n} lancements seulement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parite_des_sondes() {
|
||||
let fx = fixture();
|
||||
for (input, want) in fx["cpu_list"].as_object().unwrap() {
|
||||
let got = serve::parse_cpu_list(input);
|
||||
match want {
|
||||
Value::String(_) => assert!(got.is_none(), "{input:?}"),
|
||||
_ => assert_eq!(
|
||||
got.unwrap(),
|
||||
want.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|x| x.as_i64().unwrap())
|
||||
.collect::<Vec<_>>(),
|
||||
"{input:?}"
|
||||
),
|
||||
}
|
||||
}
|
||||
for c in fx["cpuset"].as_array().unwrap() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
for (p, d) in c["files"].as_object().unwrap() {
|
||||
let path = tmp.path().join(p);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, d.as_str().unwrap()).unwrap();
|
||||
}
|
||||
let got = serve::cpuset_threads(tmp.path());
|
||||
assert_eq!(
|
||||
(got.n, got.engine, got.why),
|
||||
(i(c, "n"), i(c, "engine"), st(c, "why")),
|
||||
"{}",
|
||||
c["name"]
|
||||
);
|
||||
}
|
||||
for c in fx["cgroup"].as_array().unwrap() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
for (p, d) in c["files"].as_object().unwrap() {
|
||||
let path = tmp.path().join(p);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, d.as_str().unwrap()).unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
serve::cgroup_mem_mib(tmp.path()),
|
||||
(i(c, "limit"), i(c, "current")),
|
||||
"{}",
|
||||
c["files"]
|
||||
);
|
||||
}
|
||||
for c in fx["smi"].as_array().unwrap() {
|
||||
assert_eq!(
|
||||
serve::vram_from_smi(&st(c, "out"), &st(c, "cvd"), b(c, "pci")),
|
||||
i(c, "mib"),
|
||||
"{c}"
|
||||
);
|
||||
}
|
||||
let devs = serve::parse_list_devices(&st(&fx["devices"], "out"));
|
||||
let want: Vec<(String, i64)> = fx["devices"]["devs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|d| (st(d, "ID"), i(d, "TotalMiB")))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
devs.iter()
|
||||
.map(|d| (d.id.clone(), d.total_mib))
|
||||
.collect::<Vec<_>>(),
|
||||
want
|
||||
);
|
||||
for (out, want) in fx["builds"].as_object().unwrap() {
|
||||
assert_eq!(
|
||||
serve::run::parse_engine_build(out),
|
||||
want.as_i64().unwrap(),
|
||||
"{out:?}"
|
||||
);
|
||||
}
|
||||
for c in fx["cmdlines"].as_array().unwrap() {
|
||||
let env: BTreeMap<String, String> = map_from(&c["env"]).into_iter().collect();
|
||||
assert_eq!(serve::engine_cmdline(&env, &strs(&c["args"])), st(c, "out"));
|
||||
}
|
||||
let cur = serve::SpecAutoMark {
|
||||
fp: "abc".into(),
|
||||
bin: "/app/llama-server".into(),
|
||||
build: 11500,
|
||||
};
|
||||
for c in fx["verdicts"].as_array().unwrap() {
|
||||
let attempt = c["attempt"].as_object().map(|a| serve::SpecAutoMark {
|
||||
fp: a["fp"].as_str().unwrap().into(),
|
||||
bin: a["bin"].as_str().unwrap().into(),
|
||||
build: a["build"].as_i64().unwrap(),
|
||||
});
|
||||
let failed = map_from(&c["failed"]);
|
||||
assert_eq!(
|
||||
serve::spec_auto_verdict(&cur, attempt.as_ref(), &failed),
|
||||
(st(c, "why"), b(c, "record")),
|
||||
"{c}"
|
||||
);
|
||||
}
|
||||
for (h, want) in fx["help_fit"].as_object().unwrap() {
|
||||
assert_eq!(
|
||||
serve::help_supports_fit(h),
|
||||
want.as_bool().unwrap(),
|
||||
"{h:?}"
|
||||
);
|
||||
}
|
||||
for (m, want) in fx["gib"].as_object().unwrap() {
|
||||
assert_eq!(
|
||||
serve::gib_text(m.parse().unwrap()),
|
||||
want.as_str().unwrap(),
|
||||
"{m}"
|
||||
);
|
||||
}
|
||||
for c in fx["load_flags"].as_array().unwrap() {
|
||||
let (out, note) = serve::normalize_load_flags(&strs(&c["args"]), b(c, "modern"));
|
||||
assert_eq!((out, note), (strs(&c["out"]), st(c, "note")), "{c}");
|
||||
}
|
||||
for c in fx["state_bytes"].as_array().unwrap() {
|
||||
let g = gguf_from(&c["gguf"]).unwrap();
|
||||
let (pt, rec) = serve::gguf_state_bytes(&g, &st(c, "k"), &st(c, "v"));
|
||||
assert_eq!(pt, c["per_token"].as_f64().unwrap(), "{c}");
|
||||
assert_eq!(rec, i(c, "recurrent"), "{c}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn port_libre() {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = l.local_addr().unwrap().port().to_string();
|
||||
let err = serve::run::wait_port_free("0.0.0.0", &port, std::time::Duration::from_millis(600))
|
||||
.unwrap_err();
|
||||
assert!(err.contains("déjà utilisé"), "{err}");
|
||||
drop(l);
|
||||
serve::run::wait_port_free("[::]", &port, std::time::Duration::from_millis(600)).unwrap();
|
||||
serve::run::wait_port_free("127.0.0.1", "", std::time::Duration::ZERO).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cle_de_persistance() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let model = tmp.path().join("m.gguf");
|
||||
std::fs::write(&model, b"x").unwrap();
|
||||
let si = SysInfo {
|
||||
model: model.to_string_lossy().into_owned(),
|
||||
..SysInfo::default()
|
||||
};
|
||||
let args: Vec<String> = ["/b", "-m", "x", "--api-key", "secret"]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
let env = BTreeMap::new();
|
||||
let k1 = serve::slot_persist_key(&args, &env, &[], &si, "/b");
|
||||
assert_eq!(k1.len(), 16);
|
||||
// La clé d'API n'entre pas dans l'empreinte.
|
||||
let mut a2 = args.clone();
|
||||
a2[4] = "autre".into();
|
||||
assert_eq!(serve::slot_persist_key(&a2, &env, &[], &si, "/b"), k1);
|
||||
// Une variable du moteur, si.
|
||||
let pe = vec![("GGML_X".to_string(), "1".to_string())];
|
||||
assert_ne!(serve::slot_persist_key(&args, &env, &pe, &si, "/b"), k1);
|
||||
assert!(serve::slot_persist_file(
|
||||
"persist-0123456789abcdef-0123456789ab.bin"
|
||||
));
|
||||
assert!(!serve::slot_persist_file(
|
||||
"persist-0123456789abcdef-0123456789ab.bak"
|
||||
));
|
||||
}
|
||||
|
||||
mod common;
|
||||
|
||||
/// Script exécutable, écrit par un `sh` à part (jamais tenu ouvert en
|
||||
/// écriture par ce process : pas d'ETXTBSY au lancement).
|
||||
#[cfg(unix)]
|
||||
fn script(dir: &std::path::Path, name: &str, body: &str) -> String {
|
||||
let p = dir.join(name);
|
||||
let ok = std::process::Command::new("sh")
|
||||
.args(["-c", "printf %s \"$1\" > \"$2\" && chmod 755 \"$2\""])
|
||||
.arg("sh")
|
||||
.arg(body)
|
||||
.arg(&p)
|
||||
.status()
|
||||
.unwrap()
|
||||
.success();
|
||||
assert!(ok);
|
||||
p.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
/// `loki-rs engine serve` de bout en bout : configuration lue par l'API
|
||||
/// interne d'un faux Go, aide d'un faux llama-server, exec de la ligne
|
||||
/// composée (le faux moteur écrit ses arguments), commande gardée en base.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn lancement_de_bout_en_bout() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let home = tmp.path().to_path_buf();
|
||||
std::fs::create_dir_all(home.join("models")).unwrap();
|
||||
std::fs::write(home.join("models/m.gguf"), b"pas un vrai gguf").unwrap();
|
||||
let out = home.join("argv.txt");
|
||||
let bin = script(
|
||||
&home,
|
||||
"llama-server",
|
||||
"#!/bin/sh\nif [ \"$1\" = \"--help\" ]; then echo '-ngl N (default: auto) --load-mode --fit [on|off]'; exit 0; fi\nprintf '%s\\n' \"$@\" > \"$OUT\"\nprintf '%s' \"$LD_LIBRARY_PATH\" > \"$OUT.ld\"\n",
|
||||
);
|
||||
let port = std::net::TcpListener::bind("127.0.0.1:0")
|
||||
.unwrap()
|
||||
.local_addr()
|
||||
.unwrap()
|
||||
.port();
|
||||
let (_facade, go, _stop) = common::fakego::start(home.clone()).await;
|
||||
go.set("config", "BIN", &bin);
|
||||
go.set("config", "MODEL", "m.gguf");
|
||||
go.set("config", "PORT", &port.to_string());
|
||||
go.set("config", "CTX", "8192");
|
||||
go.set("config", "EXTRA_ARGS", "--mlock");
|
||||
let backend = go.backend.lock().unwrap().unwrap();
|
||||
let exe = env!("CARGO_BIN_EXE_loki-rs");
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_ADDR", backend.to_string())
|
||||
.env("LOKI_INTERNAL_TOKEN", common::fakego::JETON)
|
||||
.env("OUT", &out)
|
||||
.env_remove("LLAMA_ARG_THREADS")
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
st.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&st.stderr)
|
||||
);
|
||||
let argv = std::fs::read_to_string(&out).unwrap();
|
||||
let model = home.join("models/m.gguf").to_string_lossy().into_owned();
|
||||
let want = [
|
||||
"-m",
|
||||
&model,
|
||||
"-c",
|
||||
"8192",
|
||||
"-b",
|
||||
"2048",
|
||||
"-ub",
|
||||
"512",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
&port.to_string(),
|
||||
"--parallel",
|
||||
"1",
|
||||
"-ngl",
|
||||
"auto",
|
||||
"--load-mode",
|
||||
"mmap+mlock",
|
||||
];
|
||||
let got: Vec<&str> = argv.lines().collect();
|
||||
// -t éventuel (conteneur à l'étroit) : la sonde dépend de la machine.
|
||||
let got: Vec<&str> = if got.get(4) == Some(&"-t") {
|
||||
[&got[..4], &got[6..]].concat()
|
||||
} else {
|
||||
got
|
||||
};
|
||||
assert_eq!(got, want, "{argv}");
|
||||
let ld = std::fs::read_to_string(home.join("argv.txt.ld")).unwrap();
|
||||
assert!(ld.starts_with(home.to_str().unwrap()), "{ld}");
|
||||
let cmdline = go.get("state", "engine_cmdline").unwrap();
|
||||
assert!(cmdline.starts_with(&bin), "{cmdline}");
|
||||
assert!(cmdline.contains("--load-mode mmap+mlock"), "{cmdline}");
|
||||
|
||||
// Port pris : refus en clair, code 1.
|
||||
let l = std::net::TcpListener::bind(("127.0.0.1", port)).unwrap();
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_ADDR", backend.to_string())
|
||||
.env("LOKI_INTERNAL_TOKEN", common::fakego::JETON)
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
drop(l);
|
||||
assert_eq!(st.status.code(), Some(1));
|
||||
assert!(String::from_utf8_lossy(&st.stderr).contains("déjà utilisé"));
|
||||
|
||||
// Modèle absent.
|
||||
go.set("config", "MODEL", "absent.gguf");
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_ADDR", backend.to_string())
|
||||
.env("LOKI_INTERNAL_TOKEN", common::fakego::JETON)
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(st.status.code(), Some(1));
|
||||
assert!(
|
||||
String::from_utf8_lossy(&st.stderr).contains("modèle introuvable"),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&st.stderr)
|
||||
);
|
||||
|
||||
// Preset externe : rien à lancer, sortie propre.
|
||||
go.set("config", "EXTERNAL", "1");
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_ADDR", backend.to_string())
|
||||
.env("LOKI_INTERNAL_TOKEN", common::fakego::JETON)
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(st.status.success());
|
||||
assert!(String::from_utf8_lossy(&st.stdout).contains("preset externe"));
|
||||
}
|
||||
|
||||
/// Strata, essai de l'optimiseur, pas d'API interne : `loki serve` (Go) est
|
||||
/// lancé à notre place.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn delegation_au_go() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let home = tmp.path().to_path_buf();
|
||||
let mark = home.join("go.txt");
|
||||
let go_bin = script(&home, "loki", "#!/bin/sh\necho \"go $*\" >> \"$MARK\"\n");
|
||||
let exe = env!("CARGO_BIN_EXE_loki-rs");
|
||||
// Sans API interne.
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_BIN", &go_bin)
|
||||
.env("MARK", &mark)
|
||||
.env_remove("LOKI_GO_ADDR")
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
st.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&st.stderr)
|
||||
);
|
||||
// Essai de l'optimiseur.
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_BIN", &go_bin)
|
||||
.env("MARK", &mark)
|
||||
.env("LOKI_TUNE_TRIAL", "/tmp/essai")
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(st.status.success());
|
||||
// Preset Strata.
|
||||
let (_facade, go, _stop) = common::fakego::start(home.clone()).await;
|
||||
go.set("config", "ENGINE", "strata");
|
||||
let backend = go.backend.lock().unwrap().unwrap();
|
||||
let st = tokio::process::Command::new(exe)
|
||||
.args(["engine", "serve"])
|
||||
.env("LOKI_HOME", &home)
|
||||
.env("LOKI_GO_BIN", &go_bin)
|
||||
.env("MARK", &mark)
|
||||
.env("LOKI_GO_ADDR", backend.to_string())
|
||||
.env("LOKI_INTERNAL_TOKEN", common::fakego::JETON)
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(st.status.success());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(&mark).unwrap(),
|
||||
"go serve\ngo serve\ngo serve\n"
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user