From f6c191922fe748a55e393e6ed73627087f6b5deb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 01:06:31 +0000 Subject: [PATCH] =?UTF-8?q?Lot=2028=20:=20optimiseur=20tenu=20par=20la=20f?= =?UTF-8?q?a=C3=A7ade=20(LOKI=5FTUNE=3Drust)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avec LOKI_TUNE=rust (expérimental, Unix seulement), la façade sert /api/tune, /api/tune/status, /api/tune/cancel, /api/tune/apply et /api/tune/last, et déroule l'optimisation sans perte (rust/loki-rs/src/tune) : vrai moteur arrêté, essais composés à blanc puis lancés par les « loki serve » du Go (LOKI_TUNE_TRIAL), chacun dans son groupe de processus, mesurés par le bench complet à profondeur fixe, retenus au-delà du bruit ; vrai moteur relancé. Rien n'est écrit sans un clic : copie du preset, ou le preset lui-même — redémarrage, sonde, et ancienne version rétablie si elle échoue. - tune/core.rs : la partie pure (EXTRA_ARGS, réglages, essais de chaque étape, garde-fous, diff et réécriture du preset, score) ; - tune/lock.rs : le verrou tune.lock au format du Go, identité des processus comme procIdentity (la reprise du Go reconnaît la façade vivante, et écarte son verrou si elle meurt), ménage d'un verrou périmé (essai orphelin arrêté, application interrompue défaite) ; - tune/job.rs : routes, job, essais et application. Le verrou de génération reste celui du Go, pris par le bail du moteur au motif « tune » (refus de l'optimiseur pour les autres, stop du chat qui annule). Le Go prête /internal/v1/tune/context (bench, génération, jobs du mode code, état du moteur, télémétrie) et le démarrage ou l'arrêt du moteur sans le refus de l'optimiseur (service/action « os »). Hors Unix, la façade passe LOKI_TUNE vide au Go, qui garde l'optimiseur. serde_json lit désormais les flottants à l'arrondi exact (float_roundtrip), comme le Go. Parité : parity_tune_test.go tire du Go un journal (partie pure, contenu du verrou, résultat enregistré, vérification après application) rejoué par tests/tune_core.rs. tests/tune.rs déroule de vraies optimisations, le binaire de test jouant « loki serve » ; les tests du verrou du Go sont portés ; TestFacadeTune côté Go. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016tcGfJWefW5kUGoC2wL8cv --- README.md | 1 + ...-rust-migration-lot28-optimiseur-design.md | 182 + internal/loki/facade_bench_test.go | 2 +- internal/loki/facade_chat.go | 2 + internal/loki/facade_system.go | 15 +- internal/loki/facade_tune.go | 70 + internal/loki/facade_tune_test.go | 168 + internal/loki/parity_tune_test.go | 544 + internal/loki/web_internal.go | 2 + internal/loki/web_internal_lease.go | 38 +- internal/loki/web_internal_lease_test.go | 4 +- rust/loki-rs/Cargo.toml | 2 +- rust/loki-rs/src/api.rs | 3 + rust/loki-rs/src/bench/job.rs | 2 +- rust/loki-rs/src/bench/mod.rs | 6 +- rust/loki-rs/src/enginectl.rs | 2 +- rust/loki-rs/src/lib.rs | 1 + rust/loki-rs/src/main.rs | 19 + rust/loki-rs/src/proxy.rs | 7 + rust/loki-rs/src/serve/threads.rs | 2 +- rust/loki-rs/src/tune/core.rs | 1579 + rust/loki-rs/src/tune/job.rs | 2208 ++ rust/loki-rs/src/tune/lock.rs | 615 + rust/loki-rs/src/tune/mod.rs | 15 + rust/loki-rs/src/tune/tests.rs | 294 + rust/loki-rs/tests/common/fakego.rs | 31 + rust/loki-rs/tests/lot2.rs | 1 + rust/loki-rs/tests/parity_tune/core.json | 29822 ++++++++++++++++ rust/loki-rs/tests/proxy.rs | 1 + rust/loki-rs/tests/tune.rs | 821 + rust/loki-rs/tests/tune_core.rs | 404 + 31 files changed, 36838 insertions(+), 25 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-16-rust-migration-lot28-optimiseur-design.md create mode 100644 internal/loki/facade_tune.go create mode 100644 internal/loki/facade_tune_test.go create mode 100644 internal/loki/parity_tune_test.go create mode 100644 rust/loki-rs/src/tune/core.rs create mode 100644 rust/loki-rs/src/tune/job.rs create mode 100644 rust/loki-rs/src/tune/lock.rs create mode 100644 rust/loki-rs/src/tune/mod.rs create mode 100644 rust/loki-rs/src/tune/tests.rs create mode 100644 rust/loki-rs/tests/parity_tune/core.json create mode 100644 rust/loki-rs/tests/tune.rs create mode 100644 rust/loki-rs/tests/tune_core.rs diff --git a/README.md b/README.md index 0fddd7e..2892ed8 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,7 @@ redémarrages (volume `/data`). Variables d'environnement du conteneur : | `LOKI_STRATA` | `rust` (avec `LOKI_BACKENDS=rust`) : l'écran de Strata — machine, ce qui y tient, recommandation, modèles installés —, ses réglages et son installation sont tenus par la façade Rust ; le lancement reste au Go (expérimental, migration en cours) | Go | | `LOKI_TASKS` | `rust` : les tâches planifiées — liste, enregistrement, bascule, interrupteur maître, lancement, arrêt —, le planificateur et les tâches « script seul » sont tenus par la façade Rust ; une tâche IA reste exécutée par le Go, à la demande de la façade (expérimental, migration en cours) | Go | | `LOKI_BENCH` | `rust` : le banc d'essai — lancement, progression, annulation, dernière mesure, mode rapide et mode complet en profondeur — est tenu par la façade Rust ; le verrou de génération reste celui du Go, pris par le bail du moteur (expérimental, migration en cours) | Go | +| `LOKI_TUNE` | `rust` : l'optimiseur — lancement, essais, progression, annulation, application dans une copie ou dans le preset, dernier résultat — est tenu par la façade Rust, sous Unix ; le verrou reste le fichier `tune.lock` lu par le Go, les essais des « loki serve » du Go (expérimental, migration en cours) | Go | | `LOKI_ENGINE_IMAGE` | `rust` (avec `LOKI_BACKENDS=rust`) : le panneau « moteur de l'image » — version qui tourne, dernier build publié, version recommandée, mise à jour depuis le registre officiel avec essai à blanc, bascules, retour à la version précédente, vérification du rendu du gabarit — est tenu par la façade Rust (expérimental, migration en cours) | Go | | `LOKI_PROJECTS` | `rust` : le hub des projets (liste et comptes, création, renommage, description, suppression, bascule) est tenu par la façade Rust ; les discussions restent à qui tient la conversation (expérimental, migration en cours) | Go | | `LOKI_SYSTEM` | `rust` : les interrupteurs système — accès internet et Crawl4AI, pilotage du navigateur, clé de complétion, écoute réseau du moteur — sont tenus par la façade Rust ; le navigateur piloté reste au Go (expérimental, migration en cours) | Go | diff --git a/docs/superpowers/specs/2026-10-16-rust-migration-lot28-optimiseur-design.md b/docs/superpowers/specs/2026-10-16-rust-migration-lot28-optimiseur-design.md new file mode 100644 index 0000000..80d8a28 --- /dev/null +++ b/docs/superpowers/specs/2026-10-16-rust-migration-lot28-optimiseur-design.md @@ -0,0 +1,182 @@ +# Lot 28 — Optimiseur tenu par la façade + +## Contexte + +L'optimiseur sans perte (`backend_tune*.go`) cherche des réglages du moteur +plus rapides **sans rien changer à ce que le modèle calcule** : même +contexte, mêmes slots, mêmes types de cache, même échantillonnage. Il +procède ainsi : + +1. Il arrête le vrai moteur. +2. Il lance chaque essai sur une COPIE de la configuration : + - posée dans `LOKI_HOME/tune/run//config.env`, jamais + `config.env` ; + - composée par « loki serve » à blanc ; + - puis lancée sur 127.0.0.1 et un port libre. +3. Il mesure chaque essai par le bench complet, à une profondeur fixe. +4. Il garde ce qui gagne au-delà du bruit. +5. Il relance le vrai moteur. + +Rien n'est écrit dans un preset sans un geste de l'utilisateur : dans une +copie, ou dans le preset lui-même — moteur redémarré, sondé, et ancienne +version rétablie si la sonde échoue. + +Cinq routes : + +- `POST /api/tune` lance (202) ; +- `GET /api/tune/status` dit la phase, les essais, l'ETA, puis le résultat ; +- `POST /api/tune/cancel` annule ; +- `POST /api/tune/apply` écrit le résultat, sur un clic (`copy` ou + `preset`) ; +- `GET /api/tune/last` rend le dernier résultat d'un preset, et si le preset + ou le moteur ont changé depuis. + +Le moteur, pendant une optimisation, appartient aux essais. Le verrou est un +**fichier** de `LOKI_HOME` (`tune.lock`), créé de façon exclusive, que tout +le monde lit : + +- le Go, dans tous ses refus (`tuneGuard`) ; +- « loki tune » en ligne de commande ; +- désormais la façade. + +## Le lot + +Avec `LOKI_TUNE=rust` (expérimental, Unix seulement), la façade +(`rust/loki-rs/src/tune`) sert les cinq routes et déroule l'optimisation : + +- `tune/core.rs` : la partie pure, portée à l'octet près : + - découpe et réécriture d'`EXTRA_ARGS` ; + - réglages par clé ou par drapeau ; + - essais de chaque étape selon la machine et les cases cochées ; + - garde-fous : cœur de la ligne, dénature, clés permises ; + - diff et réécriture du preset ; + - journal d'un essai ; + - score et départage ; +- `tune/lock.rs` : le verrou, **au format du Go** : + - contenu du fichier, clé de prise, binaires comparés par leur nom ; + - identité d'un processus comme `procIdentity` l'écrit : champ 22 de + `/proc//stat`, préfixé de l'identifiant du démarrage ; `ps` ailleurs ; + - écriture d'un bloc ; + - rendu neutre avant retrait ; + - ménage par déplacement-relecture ; +- `tune/job.rs` : les routes, le job, les essais et l'application. + +**La reprise reste au Go.** Sa reprise (`tuneRecoverWatch`, toutes les +30 s) lit le verrou de la façade et la reconnaît vivante, par la même +identité. Si la façade meurt en plein essai, le Go écarte son verrou comme +celui d'un « loki tune » tué : + +- l'essai orphelin est arrêté (identité complète vérifiée) ; +- une application interrompue est défaite ; +- le moteur est relancé s'il tournait. + +La façade fait le même ménage quand elle trouve un verrou périmé en prenant +le sien. + +**Les essais restent des « loki serve » du Go**, lancés par la façade avec +`LOKI_TUNE_TRIAL` (et `LOKI_TUNE_DRYRUN` à blanc), dans leur propre groupe de +processus. Le groupe est arrêté en sortie, quoi qu'il arrive. L'essai est +noté dans le verrou avant de charger quoi que ce soit. + +Le verrou de génération reste celui de la conversation du Go. La façade le +prend par le **bail du moteur**, avec le motif `tune` : + +- le Go note alors `benching` et donne aux autres le refus de l'optimiseur ; +- changer de discussion ne rompt pas le bail ; +- un stop du chat, un Reset ou un bail perdu annulent l'optimisation. + +Le Go prête à la façade ce qu'il est seul à voir +(`/internal/v1/tune/context`) : + +- **le refus du processus web** : un benchmark (le sien ou celui de la + façade), une génération, des jobs d'arrière-plan du mode code ; +- **l'état du vrai moteur** (`engineNeedsStop`) ; +- **la télémétrie**, d'où viennent les poids d'un tour type. + +Il prête aussi le démarrage et l'arrêt du moteur **sans le refus de +l'optimiseur** (`/internal/v1/service/action`, `os`), puisque c'est la +façade qui tient le verrou. Si la façade pilote elle-même le moteur +(`LOKI_ENGINE=rust`), elle passe par son superviseur. + +Le preset est réécrit et appliqué par les fonctions des presets +(`save_applying`, mode `apply` : la configuration, sans redémarrage). Le +redémarrage, la sonde et le retour arrière sont ceux de l'optimiseur. + +La ligne de commande (`loki tune`) reste au Go. + +Hors Unix, la façade ne tient pas l'optimiseur. Elle passe alors +`LOKI_TUNE` vide au Go, qui le garde et ne relaie rien. + +## Parité + +`internal/loki/parity_tune_test.go` tire du Go un journal +(`tests/parity_tune/core.json`) rejoué par `tests/tune_core.rs`. Il couvre : + +- toute la partie pure ; +- le **verrou** : contenu du fichier octet pour octet, clé de prise, + comparaison des binaires ; +- le **résultat** tel qu'enregistré et montré (listes vides écrites `null` + comme une tranche nil du Go) ; +- la vérification **après application** (la configuration est-elle l'essai + retenu ?). + +`tests/tune.rs` déroule de vraies optimisations, essais compris : le binaire +de test joue « loki serve ». + +- À blanc, il écrit la ligne de commande de l'essai. +- Pour de vrai, il sert un faux llama-server dont le prefill est d'autant + plus rapide que le micro-lot est petit. + +Les tests couvrent : + +- **une optimisation entière** : + - bail « tune » et verrou au nom de la façade ; + - vrai moteur arrêté puis relancé sans le refus de l'optimiseur, dictée + coupée ; + - quatre essais composés, lancés, mesurés, l'un retenu ; + - copies de configuration sur une adresse privée ; + - résultat enregistré par preset et rendu par `/api/tune/last` ; +- **l'application** : + - dans une copie (rien d'autre ne bouge) ; + - dans le preset (réécrit, appliqué, moteur redémarré et sondé) ; + - refusée sur un preset changé depuis la mesure ; +- **une sonde qui échoue** (contexte perdu) : preset et configuration + rétablis, moteur redémarré dessus, sauvegarde faite avant d'écrire ; +- **les essais refusés** : à la composition, au chargement (mémoire + insuffisante) ; la référence qui ne se compose pas ; la relance qui + échoue, dite dans l'erreur ; +- **l'annulation**, par la route et par le stop du chat ; +- **les refus d'entrée**, avec les messages du Go : + - preset externe, Strata, aucun preset actif ; + - jobs du mode code, moteur occupé, génération en cours ; + - optimisation d'un autre processus (et `elsewhere` dans le statut) ; +- **un verrou périmé** : + - l'essai orphelin arrêté seulement si son identité concorde ; + - une application interrompue défaite ; +- les routes sans la façade. + +Les tests du verrou du Go sont portés (`tune/tests.rs`) : + +- prise, refus, essai noté, rendu ; +- propriétaire mort ; +- verrou à notre PID qui n'est pas l'un des nôtres ; +- illisible neuf ou vieux ; +- ménage jamais sur un verrou vivant ; +- binaire remplacé ; +- identité d'un processus ; +- arrêt du groupe. + +Côté Go, `TestFacadeTune` vérifie : + +- les cinq routes partent à la façade, et elles seules ; +- le bail « tune » : refus, bascule de discussion, libération ; +- le contexte : au repos, pendant l'optimisation, pendant un bench ; +- le démarrage et l'arrêt sans le refus de l'optimiseur. + +## Écarts assumés + +- Une sortie à blanc qui échoue est lue dans un fichier où vont les deux + sorties (comme `CombinedOutput`), mais une erreur de lecture de + `launch.json` ou d'une requête garde le texte de Rust. +- Un niveau de raisonnement appris pendant les essais n'est pas transmis au + chat du Go quand le Go tient la conversation. diff --git a/internal/loki/facade_bench_test.go b/internal/loki/facade_bench_test.go index e4b085c..9fa8f5e 100644 --- a/internal/loki/facade_bench_test.go +++ b/internal/loki/facade_bench_test.go @@ -109,7 +109,7 @@ func TestFacadeBench(t *testing.T) { conv.extLeaseClearLocked() conv.Generating = false conv.mu.Unlock() - if _, err := conv.extLeaseAcquire(5*time.Second, true); err != nil { + if _, err := conv.extLeaseAcquire(5*time.Second, errBenchBusy); err != nil { t.Fatal(err) } conv.mu.Lock() diff --git a/internal/loki/facade_chat.go b/internal/loki/facade_chat.go index f2795c2..46852f2 100644 --- a/internal/loki/facade_chat.go +++ b/internal/loki/facade_chat.go @@ -183,6 +183,8 @@ func withFacadeChat(path string, h http.HandlerFunc) http.HandlerFunc { held, unavailable = facadeTasks, "tâches indisponibles : " case facadeBenchRoutes[path]: held, unavailable = facadeBench, "banc d'essai indisponible : " + case facadeTuneRoutes[path]: + held, unavailable = facadeTune, "optimiseur indisponible : " default: return h } diff --git a/internal/loki/facade_system.go b/internal/loki/facade_system.go index 4682466..93bcc37 100644 --- a/internal/loki/facade_system.go +++ b/internal/loki/facade_system.go @@ -64,6 +64,9 @@ func handleInternalCDPShutdown(w http.ResponseWriter, r *http.Request) { // internalServiceAction : serviceAction, variable pour les tests. var internalServiceAction = serviceAction +// internalServiceActionOS : serviceActionOS, variable pour les tests. +var internalServiceActionOS = func(a string) error { return serviceActionOSFn(a) } + // handleInternalServiceAction : POST {action, if_active} → 200 {ran, error}. // if_active : seulement si le moteur tourne (une clé neuve ne s'applique qu'au // redémarrage, inutile de démarrer un moteur arrêté pour elle). @@ -75,6 +78,9 @@ func handleInternalServiceAction(w http.ResponseWriter, r *http.Request) { var in struct { Action string `json:"action"` IfActive bool `json:"if_active"` + // OS : sans le refus de l'optimiseur (serviceActionOS) — pour la + // façade qui tient elle-même son verrou (LOKI_TUNE=rust). + OS bool `json:"os"` } if err := json.NewDecoder(r.Body).Decode(&in); err != nil { sendJSON(w, http.StatusBadRequest, map[string]any{"error": err.Error()}) @@ -90,9 +96,16 @@ func handleInternalServiceAction(w http.ResponseWriter, r *http.Request) { sendJSON(w, http.StatusOK, map[string]any{"ran": false, "error": ""}) return } + act := internalServiceAction + if in.OS { + act = internalServiceActionOS + } msg := "" - if err := internalServiceAction(in.Action); err != nil { + if err := act(in.Action); err != nil { msg = err.Error() } + if in.OS { + gpuMonitor.invalidate() + } sendJSON(w, http.StatusOK, map[string]any{"ran": true, "error": msg}) } diff --git a/internal/loki/facade_tune.go b/internal/loki/facade_tune.go new file mode 100644 index 0000000..50e38a7 --- /dev/null +++ b/internal/loki/facade_tune.go @@ -0,0 +1,70 @@ +package loki + +import ( + "fmt" + "net/http" + "os" + "strings" +) + +// facade_tune.go — l'optimiseur tenu par la façade Rust (LOKI_TUNE=rust, lot +// 28 de la migration ; expérimental, le Go reste le défaut) : les routes +// /api/tune*, le job, les essais et l'application (rust/loki-rs/src/tune). +// +// Le verrou de l'optimiseur reste le FICHIER de LOKI_HOME, au même format +// (backend_tune_lock.go) : le Go le lit (tuneGuard), et sa reprise +// (tuneRecoverWatch) écarte celui d'une façade morte — essai orphelin arrêté, +// application défaite, moteur relancé. Le verrou de génération : le bail du +// moteur, motif « tune » (errTuneBusy pour les autres, le stop du chat +// annule). Les essais restent des « loki serve » du Go (LOKI_TUNE_TRIAL). +// +// Le Go prête ce qu'il est seul à voir (handleInternalTuneContext) et le +// démarrage et l'arrêt du moteur SANS le refus de l'optimiseur +// (handleInternalServiceAction, « os ») : c'est la façade qui tient le verrou. + +const facadeTuneEnv = "LOKI_TUNE" + +// facadeTune : la façade tient-elle l'optimiseur ? +func facadeTune() bool { + return strings.EqualFold(strings.TrimSpace(os.Getenv(facadeTuneEnv)), "rust") && + facadeAddr() != "" && internalToken() != "" +} + +// facadeTuneRoutes : les routes que la façade sert alors. +var facadeTuneRoutes = map[string]bool{ + "/api/tune": true, + "/api/tune/status": true, + "/api/tune/cancel": true, + "/api/tune/apply": true, + "/api/tune/last": true, +} + +// handleInternalTuneContext : GET → {busy, main_active, perf}. +// +// - busy : ce qui interdit d'optimiser dans le processus web +// (tunePreflight) — un benchmark, une génération, des jobs du mode code ; +// "" sinon ; +// - main_active : le vrai moteur tourne (engineNeedsStop) ; +// - perf : la télémétrie, d'où l'optimiseur tire les poids d'un tour type. +func handleInternalTuneContext(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + sendJSON(w, http.StatusMethodNotAllowed, map[string]any{"error": "GET attendu"}) + return + } + busy := "" + switch { + case benchRunning(): + busy = errBenchBusy.Error() + case conv.isGenerating(): + busy = "génération en cours" + if err := conv.busyReason(); err != nil { + busy = err.Error() + } + case bgJobsRunning() > 0: + busy = fmt.Sprintf("%d job(s) d'arrière-plan du mode code tournent encore (bash_bg) : ils fausseraient les mesures — arrête-les d'abord", + bgJobsRunning()) + } + sendJSON(w, http.StatusOK, map[string]any{"busy": busy, "main_active": engineNeedsStop(), + "perf": perfLog.snapshot()}) +} diff --git a/internal/loki/facade_tune_test.go b/internal/loki/facade_tune_test.go new file mode 100644 index 0000000..95913d1 --- /dev/null +++ b/internal/loki/facade_tune_test.go @@ -0,0 +1,168 @@ +package loki + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// L'optimiseur part à la façade avec LOKI_TUNE=rust (lot 28) ; le reste, +// non. La façade tient le verrou de génération par le bail du moteur, motif +// « tune » ; le Go lui prête ce qu'il est seul à voir et le démarrage du +// moteur sans le refus de l'optimiseur. +func TestFacadeTune(t *testing.T) { + testHome(t) + var mu sync.Mutex + var calls []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(internalTokenHeader) != jetonTest { + w.WriteHeader(403) + return + } + mu.Lock() + calls = append(calls, r.Method+" "+r.URL.Path) + mu.Unlock() + io.WriteString(w, `{"ok":true,"de":"facade"}`) + })) + defer srv.Close() + t.Setenv(internalTokenEnv, jetonTest) + t.Setenv(facadeAddrEnv, strings.TrimPrefix(srv.URL, "http://")) + local := func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "go") } + + t.Setenv(facadeTuneEnv, "") + if facadeTune() { + t.Fatal("tenu sans LOKI_TUNE") + } + for p := range facadeTuneRoutes { + w := httptest.NewRecorder() + withFacadeChat(p, local)(w, httptest.NewRequest("POST", p, strings.NewReader("{}"))) + if w.Body.String() != "go" { + t.Fatalf("sans LOKI_TUNE, %s : %q", p, w.Body.String()) + } + } + t.Setenv(facadeTuneEnv, " RUST") + if len(facadeTuneRoutes) != 5 { + t.Fatalf("routes : %v", facadeTuneRoutes) + } + for p := range facadeTuneRoutes { + w := httptest.NewRecorder() + withFacadeChat(p, local)(w, httptest.NewRequest("POST", p, strings.NewReader("{}"))) + if !strings.Contains(w.Body.String(), "facade") { + t.Fatalf("%s non relayé : %q", p, w.Body.String()) + } + } + mu.Lock() + n := len(calls) + mu.Unlock() + if n != len(facadeTuneRoutes) { + t.Fatalf("relais : %v", calls) + } + w := httptest.NewRecorder() + withFacadeChat("/api/bench", local)(w, httptest.NewRequest("POST", "/api/bench", nil)) + if w.Body.String() != "go" { + t.Fatalf("le banc d'essai reste au Go : %q", w.Body.String()) + } + + post := func(h http.HandlerFunc, path, body string) (int, map[string]any) { + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest("POST", path, strings.NewReader(body))) + var out map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &out) + return rec.Code, out + } + tuneCtx := func() map[string]any { + rec := httptest.NewRecorder() + handleInternalTuneContext(rec, httptest.NewRequest("GET", "/internal/v1/tune/context", nil)) + if rec.Code != 200 { + t.Fatalf("contexte : %d", rec.Code) + } + var out map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + return out + } + + // Au repos : rien n'interdit, la télémétrie suit. + perfRecord(perfRec{Kind: perfMain, Complete: true, Total: 100, New: 100, Gen: 10}, nil) + c := tuneCtx() + if c["busy"] != "" || c["main_active"] != false { + t.Fatalf("contexte au repos : %v", c) + } + if perf, _ := c["perf"].([]any); len(perf) == 0 { + t.Fatalf("télémétrie absente : %v", c["perf"]) + } + rec := httptest.NewRecorder() + handleInternalTuneContext(rec, httptest.NewRequest("POST", "/internal/v1/tune/context", nil)) + if rec.Code != 405 || rec.Header().Get("Allow") != "GET" { + t.Fatalf("POST : %d", rec.Code) + } + + // Le bail « tune » : les autres reçoivent le refus de l'optimiseur ; ce + // n'est pas un benchmark. + code, out := post(handleInternalLease, "/internal/v1/engine/lease", `{"ttl_ms":60000,"why":"tune"}`) + if code != 200 { + t.Fatalf("bail : %d %v", code, out) + } + id, _ := out["lease"].(string) + if conv.busyReason() != errTuneBusy || benchRunning() || !conv.isGenerating() { + t.Fatalf("bail de l'optimiseur : refus=%v bench=%v", conv.busyReason(), benchRunning()) + } + if code, out := post(handleInternalLease, "/internal/v1/engine/lease", `{}`); code != 409 || out["error"] != errTuneBusy.Error() { + t.Fatalf("second bail : %d %v", code, out) + } + if c := tuneCtx(); c["busy"] != errTuneBusy.Error() { + t.Fatalf("contexte pendant l'optimisation : %v", c) + } + // Changer de discussion ne rompt pas le bail de l'optimiseur. + conv.loadFrom("autre", nil) + if !conv.isGenerating() || conv.busyReason() != errTuneBusy { + t.Fatal("le bail de l'optimiseur a sauté à la bascule de discussion") + } + if code, _ := post(handleInternalLeaseRelease, "/internal/v1/engine/lease/release", `{"lease":"`+id+`"}`); code != 204 { + t.Fatalf("libération : %d", code) + } + if conv.isGenerating() || conv.busyReason() == errTuneBusy { + t.Fatal("bail rendu : le verrou devait être libre") + } + // Un benchmark de la façade : le contexte le dit en premier. + code, out = post(handleInternalLease, "/internal/v1/engine/lease", `{"why":"bench"}`) + if code != 200 { + t.Fatalf("bail du bench : %d %v", code, out) + } + if c := tuneCtx(); c["busy"] != errBenchBusy.Error() { + t.Fatalf("contexte pendant un bench : %v", c) + } + post(handleInternalLeaseRelease, "/internal/v1/engine/lease/release", `{"lease":"`+out["lease"].(string)+`"}`) + + // Démarrage et arrêt sans le refus de l'optimiseur : la façade tient le + // verrou. + var guarded, osActs []string + oldA, oldOS := internalServiceAction, internalServiceActionOS + t.Cleanup(func() { internalServiceAction, internalServiceActionOS = oldA, oldOS }) + internalServiceAction = func(a string) error { guarded = append(guarded, a); return errTuneBusy } + internalServiceActionOS = func(a string) error { osActs = append(osActs, a); return nil } + for _, c := range []struct{ body, want string }{ + {`{"action":"stop","os":true}`, `{"error":"","ran":true}`}, + {`{"action":"start","os":true}`, `{"error":"","ran":true}`}, + {`{"action":"start"}`, `{"error":"` + errTuneBusy.Error() + `","ran":true}`}, + } { + w := httptest.NewRecorder() + handleInternalServiceAction(w, httptest.NewRequest("POST", "/internal/v1/service/action", strings.NewReader(c.body))) + if got := strings.TrimSpace(w.Body.String()); w.Code != 200 || got != c.want { + t.Fatalf("%s : %d %s", c.body, w.Code, got) + } + } + if strings.Join(osActs, ",") != "stop,start" || strings.Join(guarded, ",") != "start" { + t.Fatalf("actions : sans refus %v, avec %v", osActs, guarded) + } + + t.Setenv(internalTokenEnv, "") + if facadeTune() { + t.Fatal("tenu sans jeton interne") + } +} diff --git a/internal/loki/parity_tune_test.go b/internal/loki/parity_tune_test.go new file mode 100644 index 0000000..115a09b --- /dev/null +++ b/internal/loki/parity_tune_test.go @@ -0,0 +1,544 @@ +package loki + +import ( + "encoding/json" + "os" + "sort" + "strings" + "testing" +) + +// parity_tune_test.go — journal de référence de la partie PURE de +// l'optimiseur (lot 28 de la migration Rust) : découpe et réécriture +// d'EXTRA_ARGS, réglages par clé ou par drapeau, essais de chaque étape selon +// la machine et les cases cochées, garde-fous (cœur de la ligne, dénature, +// clés permises), diff et réécriture du preset, journal d'un essai, score et +// départage. Rangé dans rust/loki-rs/tests/parity_tune/core.json ; +// tests/tune_core.rs le rejoue. +// +// LOKI_PARITY_UPDATE=1 go test ./internal/loki -run TestPariteTuneCore + +const parityTuneCoreFile = "../../rust/loki-rs/tests/parity_tune/core.json" + +func parityTuneCands(c []tuneCand) []map[string]any { + out := []map[string]any{} + for _, x := range c { + out = append(out, map[string]any{"stage": x.Stage, "label": x.Label, "cfg": x.Cfg}) + } + return out +} + +func TestPariteTuneCore(t *testing.T) { + journal := map[string]any{} + + // --- EXTRA_ARGS --------------------------------------------------------- + extras := []string{ + "", " ", "-b 2048", `--chat-template-file "/mes modèles/tpl.jinja" -ub 512`, + `-ot 'blk\.(1[0-9])\.ffn_.*_exps\.=CPU' --n-cpu-moe 40`, + `-ot "blk\.([0-9]+)\.ffn_.*_exps\.=CPU,per_layer_token_embd=CPU" --cache-type-k q8_0 --n-cpu-moe 40 ` + + `--flash-attn on -ub 512 --chat-template-file "/a b/t.jinja" -ts 0.9,0.1 -ctv q8_0 -cmoe`, + "-ub=1024 -t 8 -b 4096", " -ts 0.9,0.1 --flash-attn on", "-ts=1,1 -mg 0", + "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", + "-ot=per_layer_token_embd=CPU", "--override-tensor exps=CPU", "-ot", "-t", "-t 8 -t 6", "-ub", + `"" -x`, `""x y`, `a""b`, `'a b'c "d"`, "--cache-type-k q8_0\t--flash-attn on\n-t 8", + `"non fermé -b 4`, `x"y z"w -ub=1024`, "-bs", "--backend-sampling -t 4", "-b 512\t-ub 256", + "-ncmoe", "--n-cpu-moe 0 --cpu-moe", + } + var ex []map[string]any + for _, s := range extras { + var spans [][]any + for _, sp := range argSpans(s) { + spans = append(spans, []any{sp.start, sp.end, sp.val}) + } + dropped, removed := tuneExtraDropExperts(s) + ex = append(ex, map[string]any{ + "in": s, + "spans": spans, + "split": splitArgs(s), + "set_ub": tuneExtraSet(s, knobUBatch.flags, "2048"), + "drop_ub": tuneExtraSet(s, knobUBatch.flags, ""), + "set_t": tuneExtraSet(s, knobThreads.flags, "7"), + "set_t_space": tuneExtraSet(s, knobThreads.flags, "a b"), + "set_b": tuneExtraSet(s, knobBatch.flags, "4096"), + "drop_ts": tuneExtraDrop(s, 1, "-ts", "--tensor-split"), + "drop_cmoe": tuneExtraDrop(s, 0, "-cmoe", "--cpu-moe"), + "add_bs": tuneExtraAddSwitch(s, "--backend-sampling"), + "drop_experts": dropped, + "removed": removed, + "core": tuneCoreArgs(splitArgs(s), false), + "core_optin": tuneCoreArgs(splitArgs(s), true), + }) + } + journal["extra"] = ex + + // --- réglages ------------------------------------------------------------ + cfgs := []map[string]string{ + {}, + {"UBATCH": "1024", "BATCH": "4096", "THREADS": "0", "FIT_TARGET": "1536", "SPEC_N_MAX": "2"}, + {"EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024"}, + {"EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5"}, + {"EXTRA_ARGS": "-ub 512", "UBATCH": " 128 "}, + {"EXTRA_ARGS": "-t"}, + tuneRichCfg(), + } + knobs := map[string]tuneKnob{"BATCH": knobBatch, "UBATCH": knobUBatch, "THREADS": knobThreads, + "FIT_TARGET": knobFitTarget, "SPEC_N_MAX": knobSpecN} + var kn []map[string]any + for _, c := range cfgs { + get, set, unset := map[string]string{}, map[string]any{}, map[string]any{} + for name, k := range knobs { + get[name] = k.get(c) + set[name] = k.set(c, "99") + unset[name] = k.set(c, "") + } + kn = append(kn, map[string]any{"cfg": c, "get": get, "set": set, "unset": unset}) + } + journal["knobs"] = kn + + // --- essais --------------------------------------------------------------- + type envCase struct { + name string + env tuneEnv + } + rich := tuneRichEnv() + envs := []envCase{ + {"riche", rich}, + {"sans placement", func() tuneEnv { e := tuneRichEnv(); e.Placement = false; return e }()}, + {"sans --fit", func() tuneEnv { + e := tuneRichEnv() + e.Help = strings.ReplaceAll(tuneTestHelp, "--fit", "--nofit") + return e + }()}, + {"sans --fit-target", func() tuneEnv { + e := tuneRichEnv() + e.Help = strings.ReplaceAll(tuneTestHelp, "--fit-target", "--fit-x") + return e + }()}, + {"sans nvidia-smi", func() tuneEnv { e := tuneRichEnv(); e.SMI = false; return e }()}, + {"sans opt-in", func() tuneEnv { e := tuneRichEnv(); e.OptIn = false; return e }()}, + {"une carte", func() tuneEnv { e := tuneRichEnv(); e.GPUs = 1; return e }()}, + {"trois cartes", func() tuneEnv { e := tuneRichEnv(); e.GPUs = 3; return e }()}, + {"cœurs inconnus", func() tuneEnv { e := tuneRichEnv(); e.PhysCores = 0; return e }()}, + {"deux cœurs", func() tuneEnv { e := tuneRichEnv(); e.PhysCores = 2; return e }()}, + {"environnement posé", func() tuneEnv { + e := tuneRichEnv() + e.UserEnv = map[string]string{"GGML_OP_OFFLOAD_MIN_BATCH": "64", "GGML_CUDA_GRAPH_OPT": "1"} + e.QueuesEnv = true + e.ArgEnv = map[string]string{"LLAMA_ARG_BACKEND_SAMPLING": "1"} + return e + }()}, + {"dense deux cartes", tuneEnv{Help: tuneTestHelp, GPUs: 2, VRAMMiB: 28 << 10, ModelBytes: 10 << 30, SMI: true, + PhysCores: 8, Placement: true, OptIn: true}}, + {"modèle plus gros que la VRAM", tuneEnv{Help: tuneTestHelp, GPUs: 1, VRAMMiB: 8 << 10, ModelBytes: 20 << 30, SMI: true, + PhysCores: 8}}, + {"moteur ancien", tuneEnv{GPUs: 1, SMI: true, PhysCores: 4, Placement: true, OptIn: true}}, + } + candCfgs := []struct { + name string + cfg map[string]string + }{ + {"riche", tuneRichCfg()}, + {"simple", map[string]string{"MODEL": "m.gguf", "CTX": "32768"}}, + {"contexte non chiffré", map[string]string{"MODEL": "m.gguf", "EXTRA_ARGS": "-ot exps=CPU"}}, + {"split tensor", map[string]string{"MODEL": "m.gguf", "CTX": "32768", "SPLIT_MODE": "tensor", "EXTRA_ARGS": "-ts 1,1"}}, + {"lots posés", map[string]string{"MODEL": "m.gguf", "CTX": "8192", "UBATCH": "1024", "BATCH": "2048", + "OP_OFFLOAD_MIN_BATCH": "128", "CUDA_LAUNCH_QUEUES": "4X", "THREADS": "7", "FIT_TARGET": "1536"}}, + {"spéculation", map[string]string{"MODEL": "m.gguf", "CTX": "8192", "SPEC": "draft", "SPEC_N_MAX": "4", + "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs"}}, + {"ngram", map[string]string{"MODEL": "m.gguf", "CTX": "8192", "SPEC": "ngram"}}, + {"cpu-moe", map[string]string{"MODEL": "m.gguf", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0"}}, + } + var cs []map[string]any + for _, cc := range candCfgs { + for _, ec := range envs { + stages := map[string]any{} + for _, st := range tuneStages { + c, why := st.Gen(cc.cfg, ec.env) + stages[st.Name] = map[string]any{"why": why, "cands": parityTuneCands(c)} + } + e := ec.env + cs = append(cs, map[string]any{"cfg_name": cc.name, "env_name": ec.name, "cfg": cc.cfg, + "env": map[string]any{"help": e.Help, "gpus": e.GPUs, "vram_mib": e.VRAMMiB, "model_bytes": e.ModelBytes, + "moe": e.MoE, "phys_cores": e.PhysCores, "smi": e.SMI, "arg_env": e.ArgEnv, "user_env": e.UserEnv, + "queues_env": e.QueuesEnv, "placement": e.Placement, "opt_in": e.OptIn}, + "weights_on_cpu": tuneWeightsOnCPU(cc.cfg, ec.env), + "stages": stages}) + } + } + journal["cands"] = cs + var sw []map[string]any + for _, name := range []string{"placement", "marges", "lots", "délestage", "threads", "files CUDA", "opt-in"} { + for _, only := range [][]string{nil, {"lots"}, {"Délestage"}, {"files"}, {"FILES CUDA"}, {"marges-x"}, {" threads "}, {"opt"}, {"x"}} { + sw = append(sw, map[string]any{"name": name, "only": only, "wanted": tuneStageWanted(name, only)}) + } + } + journal["stage_wanted"] = sw + + // --- garde-fous ------------------------------------------------------------ + base := tuneLaunch{Args: []string{"/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", + "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", "8192", "--flash-attn", "on"}, + Env: map[string]string{"CUDA_VISIBLE_DEVICES": "1,0", "CUDA_SCALE_LAUNCH_QUEUES": "4x"}} + ok := tuneLaunch{Args: []string{"/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", + "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", + "--cache-ram", "12000", "--flash-attn", "on", "-t", "7"}, + Env: map[string]string{"CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128"}} + clone := func(l tuneLaunch) tuneLaunch { + c := tuneLaunch{Args: append([]string(nil), l.Args...), Env: map[string]string{}} + for k, v := range l.Env { + c.Env[k] = v + } + return c + } + var dn []map[string]any + add := func(name string, b, c tuneLaunch) { + dn = append(dn, map[string]any{"name": name, "base": b, "cand": c, + "denature": tuneDenature(b, c, false), "denature_optin": tuneDenature(b, c, true), + "same_key": tuneLaunchKey(b) == tuneLaunchKey(c)}) + } + add("réglages permis", base, ok) + add("identique", base, clone(base)) + muts := map[string]func(l *tuneLaunch){ + "contexte": func(l *tuneLaunch) { l.Args[4] = "32768" }, + "cache kv": func(l *tuneLaunch) { l.Args[16] = "f16" }, + "ot": func(l *tuneLaunch) { l.Args[18] = "exps=CPU" }, + "fa": func(l *tuneLaunch) { l.Args[len(l.Args)-3] = "off" }, + "slots": func(l *tuneLaunch) { l.Args[14] = "2" }, + "cuda": func(l *tuneLaunch) { l.Env["CUDA_VISIBLE_DEVICES"] = "0,1" }, + "graphes": func(l *tuneLaunch) { l.Env["GGML_CUDA_GRAPH_OPT"] = "1" }, + "spéculation": func(l *tuneLaunch) { l.Args = append(l.Args, "--spec-draft-n-max", "4") }, + "modèle ébauche": func(l *tuneLaunch) { + l.Args = append(l.Args, "-md", "/d.gguf") + }, + "argument de plus": func(l *tuneLaunch) { l.Args = append(l.Args, "--jinja") }, + "argument de moins": func(l *tuneLaunch) { + l.Args = l.Args[:len(l.Args)-2] + }, + "port et clé": func(l *tuneLaunch) { + l.Args = append(l.Args, "--api-key", "s3cret", "--slot-save-path", "/tmp/x", "--port=9") + }, + "cache-ram": func(l *tuneLaunch) { l.Args = append(l.Args, "-cram", "4096") }, + } + names := make([]string, 0, len(muts)) + for n := range muts { + names = append(names, n) + } + sort.Strings(names) + for _, name := range names { + c := clone(ok) + muts[name](&c) + add(name, base, c) + } + journal["denature"] = dn + + rc := tuneRichCfg() + var cc []map[string]any + check := func(name string, b, c map[string]string) { + e1, e2 := tuneCfgCheck(b, c, false), tuneCfgCheck(b, c, true) + s1, s2 := "", "" + if e1 != nil { + s1 = e1.Error() + } + if e2 != nil { + s2 = e2.Error() + } + cc = append(cc, map[string]any{"name": name, "base": b, "cand": c, "check": s1, "check_optin": s2, + "diff": tuneDiff(b, c)}) + } + check("identique", rc, copyCfg(rc)) + check("lots", rc, setKey(setKey(rc, "UBATCH", "2048"), "BATCH", "4096")) + check("contexte", rc, setKey(rc, "CTX", "32768")) + check("spec", rc, setKey(rc, "SPEC", "auto")) + check("graphes", rc, setKey(rc, "CUDA_GRAPH_OPT", "on")) + check("retrait", rc, setKey(rc, "TEMP", "")) + check("extra permis", rc, knobThreads.set(rc, "4")) + check("extra interdit", rc, setKey(rc, "EXTRA_ARGS", strings.Replace(rc["EXTRA_ARGS"], "q8_0", "f16", 1))) + check("extra placement", rc, setKey(rc, "EXTRA_ARGS", func() string { s, _ := tuneExtraDropExperts(rc["EXTRA_ARGS"]); return s }())) + check("spéculation dans extra", rc, setKey(rc, "EXTRA_ARGS", rc["EXTRA_ARGS"]+" --spec-draft-n-max 4")) + check("extra vide", map[string]string{"EXTRA_ARGS": "-t 8"}, map[string]string{}) + journal["cfg_check"] = cc + + content := "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\n" + + `EXTRA_ARGS="--cache-type-k q8_0 --flash-attn on -t 8"` + "\n" + pb := parseEnv(content) + var pp []map[string]any + patch := func(name string, win map[string]string) { + out, err := tunePatchPreset(content, pb, win) + e := "" + if err != nil { + e = err.Error() + } + pp = append(pp, map[string]any{"name": name, "win": win, "out": out, "error": e}) + } + patch("threads retirés, lots", setKey(setKey(knobThreads.set(pb, ""), "UBATCH", "2048"), "BATCH", "4096")) + patch("contexte", setKey(pb, "CTX", "32768")) + patch("rien", copyCfg(pb)) + patch("opt-in", setKey(pb, "SPEC", "auto")) + patch("retrait d'EXTRA_ARGS", setKey(pb, "EXTRA_ARGS", "")) + journal["patch"] = struct { + Content string `json:"content"` + Cases []map[string]any `json:"cases"` + }{content, pp} + + // --- journal d'un essai ------------------------------------------------------ + logs := []string{ + "load_tensors: offloaded 40/49 layers to GPU\n" + + "llama_kv_cache: size = 1024.00 MiB ( 8192 cells, 32 layers, 1/1 seqs), K (q8_0): 512.00 MiB, V (q8_0): 512.00 MiB\n" + + "load_tensors: offloaded 49/49 layers to GPU\n", + "rien", + "", + "K (F16): 1 MiB, V (f16): 1 MiB\nK (q4_0): 1, V (q8_0): 2\nK (f16): x V (f16): y", + "ggml_backend_cuda_buffer_type_alloc_buffer: cudaMalloc failed: out of memory", + "common_fit: failed to fit model\n", + "llama_model_load: error loading model: bad magic\n", + "ligne 1\nderniere ligne de journal \n\n", + "\n \n", + "x\n" + strings.Repeat("é", 100), + "x\n" + strings.Repeat("a", 159) + "é" + strings.Repeat("b", 20), + } + var lf []map[string]any + for _, l := range logs { + off, total, kv := tuneLogFacts(l) + lf = append(lf, map[string]any{"log": l, "offloaded": off, "total": total, "kv": kv, "failure": tuneLoadFailure(l)}) + } + journal["logs"] = lf + + // --- score et départage ------------------------------------------------------ + intp := func(n int) *int { return &n } + var recSets []map[string]any + weights := func(name string, recs []perfRec) { + recSets = append(recSets, map[string]any{"name": name, "recs": recs, "weights": tuneWeightsFrom(recs)}) + } + weights("vide", nil) + var few []perfRec + for i := 0; i < 19; i++ { + few = append(few, perfRec{Kind: perfMain, Complete: true, New: 1000 + i, Gen: 300}) + } + weights("trop peu", few) + var many []perfRec + for i := 0; i < 30; i++ { + r := perfRec{Kind: perfMain, Complete: true, New: 500 + 37*i, Gen: 100 + 13*i, Total: 4000 + 100*i} + if i%3 == 0 { + r.Cached = intp(100) + } else { + r.Cached = intp(r.Total - 200) + } + many = append(many, r) + } + many = append(many, perfRec{Kind: perfBench, Complete: true, New: 99999, Gen: 99999}, + perfRec{Kind: perfMain, Complete: false, New: 99999, Gen: 99999}) + weights("télémétrie", many) + var bounds []perfRec + for i := 0; i < 21; i++ { + bounds = append(bounds, perfRec{Kind: perfTask, Complete: true, New: 50, Gen: 9000, Total: 100, Cached: intp(0)}) + } + weights("bornes", bounds) + var even []perfRec + for i := 0; i < 20; i++ { + even = append(even, perfRec{Kind: perfMain, Complete: true, New: 1000 + 10*i, Gen: 200 + i, Total: 2048, Cached: intp(204)}) + } + weights("pair, froid connu", even) + journal["weights"] = recSets + + w := tuneWeights{K: 2000, G: 600, FCold: 0.05} + deep := func() *benchResult { + return &benchResult{PromptPerSecond: 1000, PredictedPerSec: 50, Depth: &benchDepth{Target: 32768, + Cold: &benchTurn{PromptPerSecond: 800}, + DecodePerSec: 40, CachedPerSec: 999, + Turns: []benchTurn{{New: 2100, Cached: 30000, Expected: 30048, PromptMs: 2000}, + {New: 2100, Cached: 32300, Expected: 32348, PromptMs: 2000}}}} + } + var runs []map[string]any + score := func(name string, r *benchResult) { + run, err := tuneRunFrom(r, w) + e := "" + if err != nil { + e = err.Error() + } + runs = append(runs, map[string]any{"name": name, "bench": r, "run": run, "error": e}) + } + score("complet", deep()) + score("vide", nil) + score("sans decode", &benchResult{PromptPerSecond: 1000}) + score("rapide", &benchResult{PromptPerSecond: 1000, PredictedPerSec: 50}) + score("profondeur sautée", &benchResult{PromptPerSecond: 1000, PredictedPerSec: 50, Depth: &benchDepth{Skipped: "petit"}}) + score("partiel", func() *benchResult { r := deep(); r.Depth.Partial = "contexte plein"; return r }()) + score("sans froid", func() *benchResult { r := deep(); r.Depth.Cold = nil; return r }()) + score("froid nul", func() *benchResult { r := deep(); r.Depth.Cold.PromptPerSecond = 0; return r }()) + score("sans tours", func() *benchResult { r := deep(); r.Depth.Turns = nil; return r }()) + score("sans decode à D", func() *benchResult { r := deep(); r.Depth.DecodePerSec = 0; return r }()) + score("cache inconnu", func() *benchResult { r := deep(); r.Depth.Turns[1].Cached = -1; return r }()) + score("cache inconnu, rien d'autre", func() *benchResult { + r := deep() + r.Depth.Turns[1].Cached = -1 + r.Depth.CachedPerSec = 0 + return r + }()) + score("rien de neuf", func() *benchResult { + r := deep() + for i := range r.Depth.Turns { + r.Depth.Turns[i].New = 48 + } + return r + }()) + journal["runs"] = runs + + var wn []map[string]any + for _, c := range [][2][]float64{ + {{20, 20.4}, {19.7, 19.8}}, {{20, 20.4}, {18}}, {{20, 20.4}, {18, 18.1}}, {{18, 22}, {18.5, 18.6}}, + {{}, {1, 2}}, {{10}, {}}, {{10, 10}, {9.6, 9.6}}, {{10, 10}, {9.7, 9.7}}, {{0}, {1, 1}}, {{10, 12}, {8, 8.2, 8.1}}, + } { + gain, need, ok := tuneWins(c[0], c[1]) + wn = append(wn, map[string]any{"base": c[0], "cand": c[1], "gain": gain, "need": need, "wins": ok, + "spread_base": tuneSpread(c[0]), "mean_cand": tuneMean(c[1])}) + } + journal["wins"] = wn + var wr []map[string]any + for _, c := range []struct { + base []float64 + first float64 + }{{[]float64{20, 20.4}, 19}, {[]float64{20, 20.4}, 19.8}, {[]float64{20}, 19.39}, {[]float64{20}, 19.41}, {nil, 1}} { + wr = append(wr, map[string]any{"base": c.base, "first": c.first, "worth": tuneWorthRepeat(c.base, c.first)}) + } + journal["worth_repeat"] = wr + + var hr []map[string]any + for _, c := range []struct { + gpus []gpuStat + visible []int + }{ + {nil, nil}, + {[]gpuStat{{Total: 24576, Used: 20000}, {Total: 12288, Used: 1000}}, nil}, + {[]gpuStat{{Total: 24576, Used: 20000}, {Total: 12288, Used: 1000}}, []int{1}}, + {[]gpuStat{{Total: 0, Used: 0}, {Total: 8192, Used: 9000}}, nil}, + {[]gpuStat{{Total: 8192, Used: 100}}, []int{3}}, + } { + free, ok := tuneHeadroom(c.gpus, c.visible) + var g [][2]int + for _, x := range c.gpus { + g = append(g, [2]int{x.Total, x.Used}) + } + hr = append(hr, map[string]any{"gpus": g, "visible": c.visible, "free": free, "ok": ok}) + } + journal["headroom"] = hr + var vg []map[string]any + for _, s := range []string{"", "0", "1,0", " 2 , 3 ", "GPU-abc", "0,,1", "+1,-2"} { + vg = append(vg, map[string]any{"in": s, "out": tuneVisibleGPUs(map[string]string{"CUDA_VISIBLE_DEVICES": s})}) + } + journal["visible"] = vg + var fa []map[string]any + for _, a := range [][]string{ + {"/b", "--host", "0.0.0.0", "--port", "8080", "-m", "x"}, + {"/b", "--host=::", "--port=1", "--port", "2", "--host"}, + {"/b", "--hostname", "x", "-p", "3"}, + } { + fa = append(fa, map[string]any{"in": a, "out": tuneForceAddr(a, "127.0.0.1", "41234")}) + } + journal["force_addr"] = fa + + // --- configuration d'un essai -------------------------------------------------- + var tc []map[string]any + for _, c := range []map[string]string{ + {"MODEL": "m.gguf", "EXTRA_ARGS": `-ot "a b" -t 8`, "PORT": "41000"}, + {"MODEL": "x\ny"}, + {"NAME": "é ü", "A": " b "}, + {"A": `"q"`, "B": "c'd", "C": "#h"}, + } { + dir := t.TempDir() + err := writeTuneTrialConfig(dir, c) + e, text := "", "" + if err != nil { + e = err.Error() + } else { + b, _ := os.ReadFile(dir + "/config.env") + text = string(b) + } + tc = append(tc, map[string]any{"cfg": c, "error": e, "text": text}) + } + journal["trial_config"] = tc + + // --- Verrou : contenu du fichier, clé de prise, binaires ----------------- + owners := []tuneOwner{ + {Owner: tuneProc{PID: 42}, Via: "web", Phase: "essais", Since: 1700000000}, + {Owner: tuneProc{PID: 42, Start: "b0c1/123", Exes: []string{"/usr/bin/loki-rs"}}, Via: "apply", + Phase: "application", Since: 1700000000, Nonce: 1700000000123456789, + Trial: &tuneProc{PID: 43, Start: "b0c1/456", Exes: []string{"/usr/bin/loki", "/opt/llama-server"}}, + MainWasActive: true, Backup: "/home/l/tune/backup/Dense-1.env", PresetID: "Dense", PresetName: "Dense"}, + } + lk := []map[string]any{} + for _, o := range owners { + b, _ := json.Marshal(o) + lk = append(lk, map[string]any{"json": string(b), "key": tuneHeldKey(o)}) + } + journal["lock_owners"] = lk + se := []map[string]any{} + for _, e := range [][2]string{ + {"/usr/local/bin/loki", "/usr/local/bin/loki (deleted)"}, {"/usr/local/bin/loki", "/usr/bin/llama-server"}, + {`C:\Loki\loki.exe`, "LOKI"}, {"", "loki"}, {"/a/llama-server", "llama-server"}, {"/a/b/", "b"}, + } { + se = append(se, map[string]any{"a": e[0], "b": e[1], "same": sameExe(e[0], e[1])}) + } + journal["same_exe"] = se + + // --- Résultat tel qu'enregistré et montré -------------------------------- + run := tuneRun{TurnSec: 3.25, ColdPP: 812.5, CachedPP: 1500, TG: 42.75, ProsePP: 2000, ProseTG: 45.5, Depth: 8192} + results := []tuneResult{{}, { + PresetID: "Dense", PresetName: "Dense", BaseFP: "abc", PresetFP: "def", Engine: "b7000-x", EngineBuild: 7000, + Weights: tuneWeights{K: 2000, G: 600, FCold: 0.05, From: "défaut"}, Depth: 8192, NCtx: 16384, Slots: 1, + Baseline: tuneTrial{Stage: "référence", Label: "configuration actuelle", Status: "mesuré", + Runs: []tuneRun{run, {TurnSec: 3.5, ProsePP: 1900, ProseTG: 44}}, TurnSec: 3.375, Headroom: -1, Sec: 61.5, + Offloaded: "33/33"}, + Trials: []tuneTrial{ + {Stage: "lots", Label: "UBATCH=512 BATCH=1024", Status: "retenu", Why: "−7.3 % par tour (seuil 3.0 %)", + Runs: []tuneRun{run}, TurnSec: 3.125, Gain: 0.0734, Headroom: 512, Changes: []string{"UBATCH 2048 → 512"}, Sec: 30}, + {Stage: "lots", Label: "UBATCH=2048 BATCH=4096", Status: "écarté", Why: "dénature : x", Headroom: -1}, + }, + Best: "UBATCH=512 BATCH=1024", Gain: 0.0734, Set: map[string]string{"UBATCH": "512", "BATCH": "1024"}, + Old: map[string]string{"UBATCH": "2048", "BATCH": ""}, Changes: []string{"UBATCH 2048 → 512", "BATCH + 1024"}, + Placement: true, Partial: "annulé", Skipped: []string{"marges : une seule carte"}, Notes: []string{"n"}, + OptIn: true, At: 1700000000, + }} + rs := []string{} + for _, r := range results { + b, _ := json.Marshal(r) + rs = append(rs, string(b)) + } + journal["results"] = rs + + // --- Après application : la configuration est-elle l'essai retenu ? ------ + ab := map[string]string{"MODEL": "m.gguf", "UBATCH": "512", "EXTRA_ARGS": "-t 8"} + ares := &tuneResult{BaseFP: configFingerprint(ab), Set: map[string]string{"UBATCH": "2048", "EXTRA_ARGS": ""}, + Old: map[string]string{"UBATCH": "512", "EXTRA_ARGS": "-t 8"}} + ap := []map[string]any{} + for _, now := range []map[string]string{ + {"MODEL": "m.gguf", "UBATCH": "2048"}, {"MODEL": "m.gguf", "UBATCH": "1024"}, + {"MODEL": "autre.gguf", "UBATCH": "2048"}, {"MODEL": "m.gguf", "UBATCH": "2048", "EXTRA_ARGS": "-t 4"}, + {"MODEL": "m.gguf", "UBATCH": "2048", "PORT": "9"}, + } { + ap = append(ap, map[string]any{"now": now, "why": tuneAppliedMismatch(now, ares)}) + } + journal["applied"] = map[string]any{"base_fp": ares.BaseFP, "set": ares.Set, "old": ares.Old, "cases": ap} + + got, err := parityJSON(journal) + if err != nil { + t.Fatal(err) + } + if os.Getenv("LOKI_PARITY_UPDATE") == "1" { + if err := os.MkdirAll("../../rust/loki-rs/tests/parity_tune", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(parityTuneCoreFile, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(parityTuneCoreFile) + if err != nil { + t.Fatalf("journal de référence absent (%v) : LOKI_PARITY_UPDATE=1 pour le créer", err) + } + if string(got) != string(want) { + t.Errorf("l'optimiseur ne produit plus le journal rangé dans %s.\n"+ + "Si c'est voulu : LOKI_PARITY_UPDATE=1 go test ./internal/loki -run TestPariteTuneCore,\n"+ + "puis cargo test --test tune_core dit si le port Rust suit.", parityTuneCoreFile) + } +} diff --git a/internal/loki/web_internal.go b/internal/loki/web_internal.go index 1764e1a..3aad6e4 100644 --- a/internal/loki/web_internal.go +++ b/internal/loki/web_internal.go @@ -133,6 +133,8 @@ func withInternalAPI(next http.Handler) http.Handler { handleInternalSideJob(w, r) case "/internal/v1/bench/perf": handleInternalBenchPerf(w, r) + case "/internal/v1/tune/context": + handleInternalTuneContext(w, r) case "/internal/v1/tasks/run": handleInternalTaskRun(w, r) case "/internal/v1/push/send": diff --git a/internal/loki/web_internal_lease.go b/internal/loki/web_internal_lease.go index 0e8979e..1438fd1 100644 --- a/internal/loki/web_internal_lease.go +++ b/internal/loki/web_internal_lease.go @@ -44,9 +44,11 @@ type extLeaseState struct { stop bool ttl time.Duration timer *time.Timer - // bench : le bail d'un benchmark de la façade (LOKI_BENCH=rust) — les - // autres reçoivent errBenchBusy, et changer de discussion ne le rompt pas. - bench bool + // why : le motif d'un bail de mesure de la façade — errBenchBusy pour un + // benchmark (LOKI_BENCH=rust), errTuneBusy pour l'optimiseur + // (LOKI_TUNE=rust). Les autres reçoivent ce refus, et changer de + // discussion ne rompt pas le bail. nil : un tour ordinaire. + why error } // extLeaseClearLocked oublie le bail de la façade. c.mu détenu. @@ -57,7 +59,7 @@ var benchLeaseHeld atomic.Bool func (c *Conversation) extLeaseClearLocked() { if c.extLease != nil { - if c.extLease.bench { + if c.extLease.why == errBenchBusy { benchLeaseHeld.Store(false) } c.extLease.timer.Stop() @@ -65,8 +67,9 @@ func (c *Conversation) extLeaseClearLocked() { } } -// extLeaseAcquire prend le verrou de génération pour la façade. -func (c *Conversation) extLeaseAcquire(ttl time.Duration, bench bool) (string, error) { +// extLeaseAcquire prend le verrou de génération pour la façade ; why : voir +// extLeaseState. +func (c *Conversation) extLeaseAcquire(ttl time.Duration, why error) (string, error) { if err := tuneGuard(); err != nil { return "", err } @@ -79,12 +82,12 @@ func (c *Conversation) extLeaseAcquire(ttl time.Duration, bench bool) (string, e if err != nil { return "", err } - st := &extLeaseState{id: fmt.Sprintf("%x", b), ttl: ttl, bench: bench} + st := &extLeaseState{id: fmt.Sprintf("%x", b), ttl: ttl, why: why} st.timer = time.AfterFunc(ttl, func() { c.extLeaseExpire(st) }) c.Generating = true - if bench { - c.benching, c.leaseWhy = true, errBenchBusy - benchLeaseHeld.Store(true) + if why != nil { + c.benching, c.leaseWhy = true, why + benchLeaseHeld.Store(why == errBenchBusy) } c.cancel = func() { c.mu.Lock() @@ -104,7 +107,7 @@ func (c *Conversation) extLeaseExpire(st *extLeaseState) { } c.extLease = nil c.Generating, c.cancel = false, nil - if st.bench { + if st.why != nil { c.benching, c.leaseWhy = false, nil benchLeaseHeld.Store(false) } @@ -130,7 +133,7 @@ func (c *Conversation) extLeaseRelease(id string) { if c.extLease == nil || c.extLease.id != id { return } - if c.extLease.bench { + if c.extLease.why != nil { c.benching, c.leaseWhy = false, nil } c.extLeaseClearLocked() @@ -161,7 +164,7 @@ func readLeaseBody(w http.ResponseWriter, r *http.Request, v any) bool { func handleInternalLease(w http.ResponseWriter, r *http.Request) { var in struct { TTLms int64 `json:"ttl_ms"` - Why string `json:"why"` // "bench" : le bail d'un benchmark + Why string `json:"why"` // "bench" : un benchmark ; "tune" : l'optimiseur } if !readLeaseBody(w, r, &in) { return @@ -170,7 +173,14 @@ func handleInternalLease(w http.ResponseWriter, r *http.Request) { if in.TTLms > 0 { ttl = min(max(time.Duration(in.TTLms)*time.Millisecond, extLeaseMinTTL), extLeaseMaxTTL) } - id, err := conv.extLeaseAcquire(ttl, in.Why == "bench") + var why error + switch in.Why { + case "bench": + why = errBenchBusy + case "tune": + why = errTuneBusy + } + id, err := conv.extLeaseAcquire(ttl, why) if err != nil { if err == ErrBusy { err = conv.busyReason() // dit QUI occupe le moteur diff --git a/internal/loki/web_internal_lease_test.go b/internal/loki/web_internal_lease_test.go index d8ae60b..31b48bf 100644 --- a/internal/loki/web_internal_lease_test.go +++ b/internal/loki/web_internal_lease_test.go @@ -86,7 +86,7 @@ func TestBailInterneVerrouPartage(t *testing.T) { func TestBailInterneNeLibereQueLeSien(t *testing.T) { testHome(t) libérerVerrou(t) - id, err := conv.extLeaseAcquire(time.Minute, false) + id, err := conv.extLeaseAcquire(time.Minute, nil) if err != nil { t.Fatal(err) } @@ -110,7 +110,7 @@ func TestBailInterneNeLibereQueLeSien(t *testing.T) { func TestBailInterneExpire(t *testing.T) { testHome(t) libérerVerrou(t) - id, err := conv.extLeaseAcquire(50*time.Millisecond, false) + id, err := conv.extLeaseAcquire(50*time.Millisecond, nil) if err != nil { t.Fatal(err) } diff --git a/rust/loki-rs/Cargo.toml b/rust/loki-rs/Cargo.toml index 6e3dd4c..07cf97c 100644 --- a/rust/loki-rs/Cargo.toml +++ b/rust/loki-rs/Cargo.toml @@ -22,7 +22,7 @@ ring = { version = "0.17.14", optional = true } reqwest = { version = "0.13.5", default-features = false, features = ["rustls-no-provider", "stream", "http2"] } rustls = { version = "0.23.46", default-features = false, features = ["ring", "std", "tls12"], optional = true } serde = { version = "1.0.229", features = ["derive"] } -serde_json = { version = "1.0.154", features = ["raw_value"] } +serde_json = { version = "1.0.154", features = ["raw_value", "float_roundtrip"] } sha1 = "0.11.0" sha2 = "0.11.1" tar = { version = "0.4.46", default-features = false } diff --git a/rust/loki-rs/src/api.rs b/rust/loki-rs/src/api.rs index ba14a1d..626c5d3 100644 --- a/rust/loki-rs/src/api.rs +++ b/rust/loki-rs/src/api.rs @@ -67,6 +67,8 @@ pub struct Ctx { pub tasks: Option>, /// Banc d'essai tenu par la façade (`LOKI_BENCH=rust`). pub bench: Option>, + /// Optimiseur tenu par la façade (`LOKI_TUNE=rust`). + pub tune: Option>, /// Jeton de l'API interne : une requête qui le porte vient du Go. pub internal_token: String, } @@ -93,6 +95,7 @@ impl Ctx { || self.strata.is_some() || self.tasks.is_some() || self.bench.is_some() + || self.tune.is_some() } /// La requête vient du Go (jeton interne valide) : déjà authentifiée par diff --git a/rust/loki-rs/src/bench/job.rs b/rust/loki-rs/src/bench/job.rs index 64603d8..8f0dc39 100644 --- a/rust/loki-rs/src/bench/job.rs +++ b/rust/loki-rs/src/bench/job.rs @@ -481,7 +481,7 @@ async fn load_last(ctx: &Ctx) -> Option { /// Le modèle du preset est-il hybride (reprise du cache aux points de /// contrôle) ? -async fn model_hybrid(ctx: &Ctx, cfg: &Cfg) -> bool { +pub(crate) async fn model_hybrid(ctx: &Ctx, cfg: &Cfg) -> bool { let m = cfg.get("MODEL").map(|s| s.trim()).unwrap_or(""); if m.is_empty() { return false; diff --git a/rust/loki-rs/src/bench/mod.rs b/rust/loki-rs/src/bench/mod.rs index 774f6dc..956a042 100644 --- a/rust/loki-rs/src/bench/mod.rs +++ b/rust/loki-rs/src/bench/mod.rs @@ -44,10 +44,10 @@ const DEPTH_MAX: i64 = 32768; const DEPTH_CPU: i64 = 16384; const DEPTH_MIN: i64 = 4096; const MARGIN: i64 = 512; -const WARM_GEN: i64 = 32; +pub(crate) const WARM_GEN: i64 = 32; const SEED: i64 = 1234; const REUSE_SLACK: i64 = 16; -const SAMPLE_BYTES: usize = 160 << 10; +pub(crate) const SAMPLE_BYTES: usize = 160 << 10; const FALLBACK_RATIO: f64 = 2.5; const THRASH_BYTES: i64 = 512 << 20; @@ -62,7 +62,7 @@ pub const ERR_NO_TIMINGS: &str = pub const CORPUS: &str = include_str!("corpus.txt"); const ASK_PROSE: &str = "\n\nContinue this passage with another 1000+ words of original varied prose, mixing French and English narrative paragraphs on different topics."; -const ASK_FIRST: &str = "Résume en quelques phrases le rôle de ce fichier."; +pub(crate) const ASK_FIRST: &str = "Résume en quelques phrases le rôle de ce fichier."; const ASK_NEXT: &str = "Écris en JavaScript une fonction courte inspirée de cette partie, sans explication."; diff --git a/rust/loki-rs/src/enginectl.rs b/rust/loki-rs/src/enginectl.rs index 7bc9978..76843af 100644 --- a/rust/loki-rs/src/enginectl.rs +++ b/rust/loki-rs/src/enginectl.rs @@ -373,7 +373,7 @@ pub(crate) async fn generating(ctx: &Ctx) -> bool { /// `asrShutdownVite` : la dictée de la façade si elle la tient, sinon celle /// du Go. -async fn dictate_stop(ctx: &Ctx) { +pub(crate) async fn dictate_stop(ctx: &Ctx) { if let Some(d) = &ctx.dictate { d.shutdown_fast().await; return; diff --git a/rust/loki-rs/src/lib.rs b/rust/loki-rs/src/lib.rs index 3070278..a4bf833 100644 --- a/rust/loki-rs/src/lib.rs +++ b/rust/loki-rs/src/lib.rs @@ -47,6 +47,7 @@ pub mod system; pub mod sysupdate; pub mod tasks; pub mod tools; +pub mod tune; pub mod tz; pub mod ui; pub mod update; diff --git a/rust/loki-rs/src/main.rs b/rust/loki-rs/src/main.rs index 7b00547..4fc942a 100644 --- a/rust/loki-rs/src/main.rs +++ b/rust/loki-rs/src/main.rs @@ -186,6 +186,13 @@ fn serve() -> ExitCode { // LOKI_BENCH=rust : le banc d'essai (routes, job, protocole) ; le // verrou de génération reste celui du Go, pris par le bail du moteur. let bench_rust = held_by_facade("LOKI_BENCH"); + // LOKI_TUNE=rust : l'optimiseur (routes, job, essais, application). + // Sous Unix seulement : l'identité des processus que porte son verrou + // (procIdentity du Go) n'est pas portée ailleurs. + let tune_rust = held_by_facade("LOKI_TUNE") && cfg!(unix); + if held_by_facade("LOKI_TUNE") && !tune_rust { + eprintln!("[loki-rs] optimiseur tenu par le Go (Unix seulement)"); + } let engine_rust = held_by_facade("LOKI_ENGINE") && match loki_rs::enginectl::usable( std::env::var("LOKI_CONTAINER").ok().as_deref(), @@ -222,12 +229,17 @@ fn serve() -> ExitCode { ("LOKI_STRATA", strata_rust), ("LOKI_TASKS", tasks_rust), ("LOKI_BENCH", bench_rust), + ("LOKI_TUNE", tune_rust), ]; for (var, on) in held { if on { env.push((var.into(), "rust".into())); } } + // Demandé mais pas tenu : le Go ne doit pas relayer ces routes. + if !tune_rust && held_by_facade("LOKI_TUNE") { + env.push(("LOKI_TUNE".into(), String::new())); + } if held.iter().any(|(_, on)| *on) { env.push(("LOKI_FACADE_ADDR".into(), format!("127.0.0.1:{web_port}"))); } @@ -357,6 +369,12 @@ fn serve() -> ExitCode { .then(|| std::sync::Arc::new(loki_rs::tasks::Tasks::new(engine_paths.home.clone()))); let bench = bench_rust .then(|| std::sync::Arc::new(loki_rs::bench::job::Bench::new(engine_paths.clone()))); + let tune = tune_rust.then(|| { + std::sync::Arc::new(loki_rs::tune::job::Tune::new( + engine_paths.home.clone(), + go_bin_path.clone(), + )) + }); let cfg = proxy::Config { backend, internal_token: token, @@ -380,6 +398,7 @@ fn serve() -> ExitCode { strata, tasks, bench, + tune, mcp: mcp_rust.then(|| std::sync::Arc::new(loki_rs::mcp::Mcp::new(mem_home))), }; let res = proxy::serve(listener, cfg, shutdown_signal()).await; diff --git a/rust/loki-rs/src/proxy.rs b/rust/loki-rs/src/proxy.rs index 5311db4..49a5f37 100644 --- a/rust/loki-rs/src/proxy.rs +++ b/rust/loki-rs/src/proxy.rs @@ -109,6 +109,7 @@ pub struct Config { /// Banc d'essai (routes, job, protocole de mesure) tenu par la façade /// (`LOKI_BENCH=rust`) ; `None` : au Go. pub bench: Option>, + pub tune: Option>, } /// Arrête la tâche de fond tenue à sa sortie de portée. @@ -164,6 +165,7 @@ pub async fn serve( strata: cfg.strata, tasks: cfg.tasks, bench: cfg.bench, + tune: cfg.tune, internal_token: cfg.internal_token.clone(), }, }); @@ -296,6 +298,11 @@ impl Proxy { { return crate::bench::job::serve(&self.ctx, b, req, from_go).await; } + if let Some(t) = &self.ctx.tune + && crate::tune::job::handles(path) + { + return crate::tune::job::serve(&self.ctx, t, req, from_go).await; + } if let Some(b) = &self.ctx.backends && crate::backends::handles(path) { diff --git a/rust/loki-rs/src/serve/threads.rs b/rust/loki-rs/src/serve/threads.rs index 0d8bbcf..0051b56 100644 --- a/rust/loki-rs/src/serve/threads.rs +++ b/rust/loki-rs/src/serve/threads.rs @@ -219,7 +219,7 @@ fn ceil_quota(quota: &str, period: &str) -> i64 { } } -fn read_cpu_list(root: &Path, rel: &str) -> Option> { +pub(crate) fn read_cpu_list(root: &Path, rel: &str) -> Option> { parse_cpu_list(&read(root, rel)?) } diff --git a/rust/loki-rs/src/tune/core.rs b/rust/loki-rs/src/tune/core.rs new file mode 100644 index 0000000..2760add --- /dev/null +++ b/rust/loki-rs/src/tune/core.rs @@ -0,0 +1,1579 @@ +//! Optimiseur sans perte — la partie PURE (port de `backend_tune.go`) : ce +//! qu'on a le droit de changer, les essais à tenter, la comparaison des +//! lignes de commande, le score, la réécriture du preset. +//! +//! Seuls des réglages d'ordonnancement bougent (placement, lots, threads, +//! files CUDA ; spéculation et graphes CUDA sur demande) : jamais le modèle, +//! le contexte, les types du cache KV, le raisonnement, l'échantillonnage, +//! ni un drapeau d'EXTRA_ARGS hors de la liste blanche. Un essai dont la +//! ligne de commande diffère de la référence ailleurs que sur les drapeaux +//! réglés est écarté (« dénature ») avant même d'être chargé. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use serde::{Deserialize, Serialize}; + +use crate::bench::{BenchResult, ser_go_f64}; +use crate::models::split_args; +use crate::presets_api::join_args; +use crate::serve::{SysInfo, flag_value, has_any_flag}; + +pub type Cfg = HashMap; + +/// `tuneTrialEnv` : le dossier de l'essai, où `loki serve` lit sa +/// configuration. +pub const TRIAL_ENV: &str = "LOKI_TUNE_TRIAL"; +/// `tuneDryRunEnv` : `loki serve` compose la ligne, l'écrit, et s'arrête. +pub const DRY_RUN_ENV: &str = "LOKI_TUNE_DRYRUN"; + +/// `tuneKeys` : les clés que l'optimiseur peut écrire. +pub const KEYS: [&str; 8] = [ + "BATCH", + "UBATCH", + "THREADS", + "THREADS_BATCH", + "FIT_TARGET", + "OP_OFFLOAD_MIN_BATCH", + "CUDA_LAUNCH_QUEUES", + "EXTRA_ARGS", +]; + +/// `tuneOptInKeys`. +pub const OPT_IN_KEYS: [&str; 3] = ["SPEC", "SPEC_N_MAX", "CUDA_GRAPH_OPT"]; + +/// `tuneNeverKeys` : ce que l'optimiseur ne touche jamais. +pub const NEVER_KEYS: [&str; 27] = [ + "MODEL", + "CTX", + "KV_TYPE", + "KV_TYPE_K", + "KV_TYPE_V", + "MMPROJ", + "REASONING", + "REASONING_BUDGET", + "REASONING_EFFORT", + "REASONING_PRESERVE", + "REASONING_ECHO", + "TEMP", + "TOP_P", + "TOP_K", + "MIN_P", + "PRESENCE_PENALTY", + "REPEAT_PENALTY", + "PARALLEL", + "SIDE_SLOT", + "MODEL_DRAFT", + "SPEC_SAMPLING", + "NGL", + "SPLIT_MODE", + "CUDA_VISIBLE_DEVICES", + "BIN", + "HOST", + "PORT", +]; + +fn is_key(k: &str) -> bool { + KEYS.contains(&k) +} + +fn is_opt_in_key(k: &str) -> bool { + OPT_IN_KEYS.contains(&k) +} + +/// `tuneFlag` : un drapeau d'EXTRA_ARGS que l'optimiseur peut retirer ou +/// régler. +pub struct Flag { + pub names: &'static [&'static str], + /// 0 = interrupteur, 1 = une valeur. + pub arity: usize, + pub opt_in: bool, +} + +/// `tuneFlags` : la liste blanche. +pub const FLAGS: [Flag; 14] = [ + Flag { + names: &["-ot", "--override-tensor"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-ncmoe", "--n-cpu-moe"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-cmoe", "--cpu-moe"], + arity: 0, + opt_in: false, + }, + Flag { + names: &["-sm", "--split-mode"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-ts", "--tensor-split"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-mg", "--main-gpu"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-fit", "--fit"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-fitt", "--fit-target"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-b", "--batch-size"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-ub", "--ubatch-size"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-t", "--threads"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["-tb", "--threads-batch"], + arity: 1, + opt_in: false, + }, + Flag { + names: &["--spec-draft-n-max", "--draft-max", "--draft-n"], + arity: 1, + opt_in: true, + }, + Flag { + names: &["-bs", "--backend-sampling"], + arity: 0, + opt_in: true, + }, +]; + +/// `tuneForbiddenFlags` : jamais ajoutés, retirés ni modifiés. +pub const FORBIDDEN_FLAGS: [&str; 34] = [ + "-ctk", + "--cache-type-k", + "-ctv", + "--cache-type-v", + "-fa", + "--flash-attn", + "-c", + "--ctx-size", + "--temp", + "--top-k", + "--top-p", + "--min-p", + "--samplers", + "--repeat-penalty", + "--repeat-last-n", + "--dry-multiplier", + "--reasoning", + "--reasoning-budget", + "--reasoning-format", + "--chat-template", + "--chat-template-file", + "--jinja", + "--cache-reuse", + "--context-shift", + "--swa-full", + "-kvu", + "--kv-unified", + "--keep", + "-np", + "--parallel", + "-m", + "--model", + "--mmproj", + "-ngl", +]; + +/// `tuneFlagFor`. +pub fn flag_for(name: &str, opt_in: bool) -> Option<&'static Flag> { + FLAGS + .iter() + .filter(|f| opt_in || !f.opt_in) + .find(|f| f.names.contains(&name)) +} + +// --------------------------------------------------------------------------- +// EXTRA_ARGS jeton par jeton +// --------------------------------------------------------------------------- + +/// `argSpan` : un argument et sa place exacte dans la chaîne. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArgSpan { + pub start: usize, + pub end: usize, + pub val: String, +} + +/// `argSpans` : découpe comme `splitArgs`, positions gardées. +pub fn arg_spans(s: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quote: Option = None; + let mut start: Option = None; + let flush = + |out: &mut Vec, cur: &mut String, start: &mut Option, end: usize| { + if !cur.is_empty() { + out.push(ArgSpan { + start: start.unwrap_or(0), + end, + val: std::mem::take(cur), + }); + } + *start = None; + }; + for (i, r) in s.char_indices() { + match quote { + Some(q) => { + if r == q { + quote = None; + if cur.is_empty() { + // « "" » : un argument vide explicite. + out.push(ArgSpan { + start: start.unwrap_or(i), + end: i + 1, + val: String::new(), + }); + start = None; + } + } else { + cur.push(r); + } + } + None if r == '"' || r == '\'' => { + if start.is_none() { + start = Some(i); + } + quote = Some(r); + } + None if matches!(r, ' ' | '\t' | '\n' | '\r') => { + flush(&mut out, &mut cur, &mut start, i); + } + None => { + if start.is_none() { + start = Some(i); + } + cur.push(r); + } + } + } + flush(&mut out, &mut cur, &mut start, s.len()); + out +} + +/// `extraFlagSpan` : une occurrence d'un drapeau, avec sa valeur éventuelle. +#[derive(Clone, Debug)] +pub struct FlagSpan { + pub name: String, + pub flag: ArgSpan, + pub value: Option, + pub eq_val: String, + pub has_eq: bool, + pub tokens: Vec, +} + +/// `extraFlags`. +pub fn extra_flags(spans: &[ArgSpan], arity: usize, names: &[&str]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while i < spans.len() { + let (name, val, has_eq) = match spans[i].val.split_once('=') { + Some((n, v)) => (n.to_string(), v.to_string(), true), + None => (spans[i].val.clone(), String::new(), false), + }; + if !names.contains(&name.as_str()) { + i += 1; + continue; + } + let mut f = FlagSpan { + name, + flag: spans[i].clone(), + value: None, + eq_val: val, + has_eq, + tokens: vec![spans[i].clone()], + }; + if arity == 1 && !has_eq && i + 1 < spans.len() { + f.value = Some(spans[i + 1].clone()); + f.tokens.push(spans[i + 1].clone()); + i += 1; + } + out.push(f); + i += 1; + } + out +} + +/// `spliceEdit` : un morceau remplacé (`repl` vide = retiré). +#[derive(Clone, Debug)] +pub struct Edit { + pub start: usize, + pub end: usize, + pub repl: String, +} + +/// `applyEdits` : remplacements disjoints ; un retrait emporte les blancs +/// qui le précèdent (ou, en tête, ceux qui le suivent). +pub fn apply_edits(s: &str, mut edits: Vec) -> String { + if edits.is_empty() { + return s.to_string(); + } + edits.sort_by_key(|e| e.start); + let b = s.as_bytes(); + let mut out = String::new(); + let mut last = 0; + for e in edits { + let (mut start, mut end) = (e.start, e.end); + if e.repl.is_empty() { + let mut ws = start; + while ws > last && (b[ws - 1] == b' ' || b[ws - 1] == b'\t') { + ws -= 1; + } + if ws == start || ws == 0 { + while end < b.len() && (b[end] == b' ' || b[end] == b'\t') { + end += 1; + } + if ws == 0 { + start = 0; + } + } else { + start = ws; + } + } + if start < last { + start = last; + } + out.push_str(&s[last..start]); + out.push_str(&e.repl); + last = end; + } + out.push_str(&s[last..]); + out.trim().to_string() +} + +/// `quoteArg`. +pub fn quote_arg(a: &str) -> String { + join_args(&[a.to_string()]) +} + +/// `tuneExtraSet` : la DERNIÈRE occurrence voit sa valeur remplacée sur +/// place ; absent, le drapeau est ajouté ; `val` vide retire tout. +pub fn extra_set(s: &str, names: &[&str], val: &str) -> String { + let occ = extra_flags(&arg_spans(s), 1, names); + if val.is_empty() { + let edits = occ + .iter() + .flat_map(|o| o.tokens.iter()) + .map(|t| Edit { + start: t.start, + end: t.end, + repl: String::new(), + }) + .collect(); + return apply_edits(s, edits); + } + let Some(o) = occ.last() else { + let add = format!("{} {}", names[0], quote_arg(val)); + if s.trim().is_empty() { + return add; + } + return format!("{} {add}", s.trim_end_matches([' ', '\t'])); + }; + if o.has_eq { + return apply_edits( + s, + vec![Edit { + start: o.flag.start, + end: o.flag.end, + repl: format!("{}={}", o.name, quote_arg(val)), + }], + ); + } + if let Some(v) = &o.value { + return apply_edits( + s, + vec![Edit { + start: v.start, + end: v.end, + repl: quote_arg(val), + }], + ); + } + format!( + "{} {}{}", + &s[..o.flag.end], + quote_arg(val), + &s[o.flag.end..] + ) +} + +/// `tuneExtraDrop` : retire toutes les occurrences (valeur comprise). +pub fn extra_drop(s: &str, arity: usize, names: &[&str]) -> String { + let edits = extra_flags(&arg_spans(s), arity, names) + .iter() + .flat_map(|o| o.tokens.clone()) + .map(|t| Edit { + start: t.start, + end: t.end, + repl: String::new(), + }) + .collect(); + apply_edits(s, edits) +} + +/// `tuneExtraAddSwitch`. +pub fn extra_add_switch(s: &str, name: &str) -> String { + if s.trim().is_empty() { + return name.to_string(); + } + format!("{} {name}", s.trim_end_matches([' ', '\t'])) +} + +/// `tuneExtraDropExperts` : retire le placement MANUEL des experts ; rend +/// aussi ce qui a été retiré. +pub fn extra_drop_experts(s: &str) -> (String, Vec) { + let spans = arg_spans(s); + let mut edits = Vec::new(); + let mut removed = Vec::new(); + for o in extra_flags(&spans, 1, &["-ot", "--override-tensor"]) { + let v = if o.has_eq { + o.eq_val.clone() + } else { + match &o.value { + Some(v) => v.val.clone(), + None => continue, + } + }; + let mut rest = Vec::new(); + let mut hit = false; + for el in v.split(',') { + let el = el.trim(); + let pat = el.split_once('=').map_or(el, |(p, _)| p); + if el.is_empty() { + } else if crate::serve::moe::ot_targets_experts(pat) { + removed.push(format!("{} {el}", o.name)); + hit = true; + } else { + rest.push(el.to_string()); + } + } + if !hit { + continue; + } + if rest.is_empty() { + for t in &o.tokens { + edits.push(Edit { + start: t.start, + end: t.end, + repl: String::new(), + }); + } + } else if o.has_eq { + edits.push(Edit { + start: o.flag.start, + end: o.flag.end, + repl: format!("{}={}", o.name, quote_arg(&rest.join(","))), + }); + } else if let Some(v) = &o.value { + edits.push(Edit { + start: v.start, + end: v.end, + repl: quote_arg(&rest.join(",")), + }); + } + } + for o in extra_flags(&spans, 1, &["-ncmoe", "--n-cpu-moe"]) { + let v = match &o.value { + Some(v) => v.val.clone(), + None => o.eq_val.clone(), + }; + removed.push(format!("{} {v}", o.name)); + for t in &o.tokens { + edits.push(Edit { + start: t.start, + end: t.end, + repl: String::new(), + }); + } + } + for o in extra_flags(&spans, 0, &["-cmoe", "--cpu-moe"]) { + removed.push(o.name.clone()); + edits.push(Edit { + start: o.flag.start, + end: o.flag.end, + repl: String::new(), + }); + } + (apply_edits(s, edits), removed) +} + +// --------------------------------------------------------------------------- +// Réglages : par la clé, ou par le drapeau qui l'écrase +// --------------------------------------------------------------------------- + +/// `tuneKnob`. +#[derive(Clone, Copy, Debug)] +pub struct Knob { + pub key: &'static str, + pub flags: &'static [&'static str], +} + +pub const KNOB_BATCH: Knob = Knob { + key: "BATCH", + flags: &["-b", "--batch-size"], +}; +pub const KNOB_UBATCH: Knob = Knob { + key: "UBATCH", + flags: &["-ub", "--ubatch-size"], +}; +pub const KNOB_THREADS: Knob = Knob { + key: "THREADS", + flags: &["-t", "--threads"], +}; +pub const KNOB_FIT_TARGET: Knob = Knob { + key: "FIT_TARGET", + flags: &["-fitt", "--fit-target"], +}; +pub const KNOB_SPEC_N: Knob = Knob { + key: "SPEC_N_MAX", + flags: &["--spec-draft-n-max"], +}; + +fn get<'a>(cfg: &'a Cfg, k: &str) -> &'a str { + cfg.get(k).map(String::as_str).unwrap_or("") +} + +impl Knob { + /// La valeur effective (drapeau d'EXTRA_ARGS, sinon la clé). + pub fn get(&self, cfg: &Cfg) -> String { + let v = flag_value(&split_args(get(cfg, "EXTRA_ARGS")), self.flags); + if !v.is_empty() { + return v; + } + get(cfg, self.key).trim().to_string() + } + + /// Une COPIE de cfg où la valeur vaut `val` ("" = retirée). + pub fn set(&self, cfg: &Cfg, val: &str) -> Cfg { + let mut out = cfg.clone(); + if has_any_flag(&split_args(get(cfg, "EXTRA_ARGS")), self.flags) { + let e = extra_set(get(cfg, "EXTRA_ARGS"), self.flags, val); + if e.is_empty() { + out.remove("EXTRA_ARGS"); + } else { + out.insert("EXTRA_ARGS".into(), e); + } + return out; + } + if val.is_empty() { + out.remove(self.key); + } else { + out.insert(self.key.into(), val.into()); + } + out + } +} + +/// `setKey` : copie de cfg avec key = val ("" = retirée). +pub fn set_key(cfg: &Cfg, key: &str, val: &str) -> Cfg { + let mut out = cfg.clone(); + if val.is_empty() { + out.remove(key); + } else { + out.insert(key.into(), val.into()); + } + out +} + +fn first_non_empty(a: &str, b: &str) -> String { + if a.trim().is_empty() { + b.to_string() + } else { + a.to_string() + } +} + +// --------------------------------------------------------------------------- +// Les essais +// --------------------------------------------------------------------------- + +/// `tuneEnv` : ce que l'optimiseur sait de la machine et de ce que +/// l'utilisateur a coché. +#[derive(Clone, Debug, Default)] +pub struct Env { + pub help: String, + pub gpus: i64, + pub vram_mib: i64, + pub model_bytes: i64, + pub moe: bool, + pub phys_cores: i64, + pub smi: bool, + pub arg_env: HashMap, + pub user_env: HashMap, + pub queues_env: bool, + pub placement: bool, + pub opt_in: bool, +} + +/// `tuneCand` : un essai — la configuration COMPLÈTE à lancer. +#[derive(Clone, Debug, PartialEq)] +pub struct Cand { + pub stage: String, + pub label: String, + pub cfg: Cfg, +} + +fn cand(stage: &str, label: impl Into, cfg: Cfg) -> Cand { + Cand { + stage: stage.into(), + label: label.into(), + cfg, + } +} + +type Gen = fn(&Cfg, &Env) -> (Vec, String); + +/// `tuneStages` : l'ordre de la recherche. +pub const STAGES: [(&str, Gen); 7] = [ + ("placement", placement_cands), + ("marges", fit_target_cands), + ("lots", batch_cands), + ("délestage", offload_cands), + ("threads", thread_cands), + ("files CUDA", queue_cands), + ("opt-in", opt_in_cands), +]; + +/// `tuneStageWanted` : noms comparés sans accent ni casse, par leur premier +/// mot. +pub fn stage_wanted(name: &str, only: &[String]) -> bool { + let norm = |s: &str| { + let s = s.trim().to_lowercase().replace(['é', 'è', 'ê'], "e"); + match s.find([' ', '-']) { + Some(i) if i > 0 && s != "opt-in" => s[..i].to_string(), + _ => s, + } + }; + let n = norm(name); + if only.is_empty() || n == "placement" || n == "opt-in" { + return true; + } + only.iter().any(|o| norm(o) == n) +} + +/// `tuneSI`. +pub fn sys_info(env: &Env) -> SysInfo { + SysInfo { + help: env.help.clone(), + arg_env: env.arg_env.clone(), + user_env: env.user_env.clone(), + ..Default::default() + } +} + +/// `tuneWeightsOnCPU`. +pub fn weights_on_cpu(cfg: &Cfg, env: &Env) -> bool { + let extra = split_args(get(cfg, "EXTRA_ARGS")); + if !crate::serve::cache::cpu_weights(cfg, &extra, &env.arg_env, 0).is_empty() { + return true; + } + env.vram_mib > 0 && env.model_bytes > env.vram_mib << 20 +} + +/// `tunePlacementCands` : le placement manuel remplacé par --fit. +pub fn placement_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + if !env.placement { + return (vec![], "non demandé (« inclure le placement »)".into()); + } + let extra = split_args(get(cfg, "EXTRA_ARGS")); + if !crate::serve::moe::help_supports_fit(&env.help) { + return (vec![], "ce moteur ne connaît pas --fit".into()); + } + let (c, ok) = crate::serve::spec::ctx_fixed(cfg, &extra); + if !ok { + return ( + vec![], + format!("contexte « {c} » non chiffré : --fit pourrait le réduire"), + ); + } + if crate::serve::split::split_mode_key(cfg) == "tensor" { + return (vec![], "SPLIT_MODE=tensor".into()); + } + let moe = !crate::serve::moe::cpu_experts(&extra, &env.arg_env).is_empty(); + let ts = has_any_flag(&extra, &["-ts", "--tensor-split"]) && env.gpus >= 2; + if !moe && !ts { + return (vec![], "rien n'est placé à la main".into()); + } + let mut e = get(cfg, "EXTRA_ARGS").to_string(); + let mut label = String::from("placement auto (--fit)"); + if moe { + e = extra_drop_experts(&e).0; + label.push_str(" au lieu des experts placés à la main"); + } + if ts { + e = extra_drop(&e, 1, &["-ts", "--tensor-split"]); + label.push_str(" au lieu de --tensor-split"); + } + let nc = set_key(cfg, "EXTRA_ARGS", &e); + let why = crate::serve::expert::fit_blocker(&nc, &split_args(&e), &sys_info(env)); + if !why.is_empty() { + return (vec![], format!("--fit resterait inactif ({why})")); + } + (vec![cand("placement", label, nc)], String::new()) +} + +/// `tuneFitTargetCands` : la marge que --fit laisse libre par carte. +pub fn fit_target_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + let extra = split_args(get(cfg, "EXTRA_ARGS")); + if env.gpus < 2 { + return (vec![], "une seule carte".into()); + } + if !env.smi { + return (vec![], "VRAM libre non mesurable (nvidia-smi)".into()); + } + if !env.help.contains("--fit-target") { + return (vec![], "ce moteur ne connaît pas --fit-target".into()); + } + if !crate::serve::spec::ctx_fixed(cfg, &extra).1 { + return (vec![], "contexte non chiffré".into()); + } + let why = crate::serve::expert::fit_blocker(cfg, &extra, &sys_info(env)); + if !why.is_empty() { + return (vec![], format!("--fit inactif ({why})")); + } + let vals: &[&str] = if env.gpus == 2 { + &["1024,2048", "2048,1024", "1536"] + } else { + &["1536", "2048"] + }; + let cur = KNOB_FIT_TARGET.get(cfg); + let out = vals + .iter() + .filter(|v| **v != cur) + .map(|v| { + cand( + "marges", + format!("FIT_TARGET={v}"), + KNOB_FIT_TARGET.set(cfg, v), + ) + }) + .collect(); + (out, String::new()) +} + +/// `strconv.Atoi` (0 sur erreur, comme `curB, _ := strconv.Atoi(...)`). +fn atoi0(s: &str) -> i64 { + let digits = s.strip_prefix(['+', '-']).unwrap_or(s); + if digits.is_empty() || !digits.bytes().all(|c| c.is_ascii_digit()) { + return 0; + } + s.parse().unwrap_or(0) +} + +/// `tuneBatchCands` : micro-lot et lot. +pub fn batch_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + let extra = split_args(get(cfg, "EXTRA_ARGS")); + let cur_ub = crate::serve::moe::serve_ubatch(cfg, &extra); + let cur_b = atoi0(&first_non_empty(&KNOB_BATCH.get(cfg), "2048")); + let mut ubs = vec![512i64, 1024, 2048]; + if env.moe { + ubs.push(4096); + } + let multi = env.gpus >= 2 && !weights_on_cpu(cfg, env); + let mut out = Vec::new(); + for ub in ubs { + if !env.smi && ub > cur_ub { + continue; + } + let mut seen = BTreeSet::new(); + for b in [2048.max(ub), (2 * ub).min(8192)] { + if seen.contains(&b) || b < ub || (multi && b <= ub) || (ub == cur_ub && b == cur_b) { + continue; + } + seen.insert(b); + let c = KNOB_UBATCH.set(cfg, &ub.to_string()); + let c = KNOB_BATCH.set(&c, &b.to_string()); + out.push(cand("lots", format!("UBATCH={ub} BATCH={b}"), c)); + } + } + if out.is_empty() { + return (vec![], "rien à essayer".into()); + } + (out, String::new()) +} + +/// `tuneOffloadCands` : seuil d'envoi au GPU des poids restés en RAM. +pub fn offload_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + if !weights_on_cpu(cfg, env) { + return (vec![], "tous les poids sur GPU".into()); + } + if env + .user_env + .get("GGML_OP_OFFLOAD_MIN_BATCH") + .is_some_and(|v| !v.is_empty()) + { + return ( + vec![], + "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement".into(), + ); + } + let cur = first_non_empty(get(cfg, "OP_OFFLOAD_MIN_BATCH").trim(), "32"); + let out = ["32", "128", "512"] + .iter() + .filter(|v| **v != cur) + .map(|v| { + cand( + "délestage", + format!("OP_OFFLOAD_MIN_BATCH={v}"), + set_key(cfg, "OP_OFFLOAD_MIN_BATCH", v), + ) + }) + .collect(); + (out, String::new()) +} + +/// `tuneThreadCands` : threads CPU, quand des poids tournent sur CPU. +pub fn thread_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + if !weights_on_cpu(cfg, env) { + return (vec![], "tous les poids sur GPU".into()); + } + if env.phys_cores <= 1 { + return (vec![], "nombre de cœurs physiques inconnu".into()); + } + let mut cur = KNOB_THREADS.get(cfg); + if cur == "0" { + cur.clear(); + } + let mut seen = BTreeSet::from([cur]); + let mut out = Vec::new(); + for n in [0, env.phys_cores - 1, env.phys_cores / 2] { + let v = if n > 0 { n.to_string() } else { String::new() }; + if seen.contains(&v) || n < 0 { + continue; + } + seen.insert(v.clone()); + let label = if v.is_empty() { + "threads omis (cœurs physiques)".to_string() + } else { + format!("THREADS={v}") + }; + out.push(cand("threads", label, KNOB_THREADS.set(cfg, &v))); + } + (out, String::new()) +} + +/// `tuneQueueCands` : files de lancement CUDA, sur deux cartes ou plus. +pub fn queue_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + if env.gpus < 2 { + return (vec![], "une seule carte".into()); + } + if env.queues_env { + return ( + vec![], + "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement".into(), + ); + } + let cur = get(cfg, "CUDA_LAUNCH_QUEUES").trim().to_lowercase(); + let out = ["off", "4x"] + .iter() + .filter(|v| **v != cur) + .map(|v| { + cand( + "files CUDA", + format!("CUDA_LAUNCH_QUEUES={v}"), + set_key(cfg, "CUDA_LAUNCH_QUEUES", v), + ) + }) + .collect(); + (out, String::new()) +} + +/// `tuneOptInCands` : spéculation, graphes CUDA, échantillonnage sur GPU — +/// seulement si l'utilisateur l'a coché. +pub fn opt_in_cands(cfg: &Cfg, env: &Env) -> (Vec, String) { + if !env.opt_in { + return ( + vec![], + "non demandé (« inclure les options opt-in »)".into(), + ); + } + let extra = split_args(get(cfg, "EXTRA_ARGS")); + let mut out = Vec::new(); + let mode = crate::serve::spec::spec_mode(cfg); + if mode == "off" + && crate::serve::spec::help_supports_mtp(&env.help) + && !crate::serve::spec::spec_user_set(&extra, &env.arg_env) + { + out.push(cand("opt-in", "SPEC=auto", set_key(cfg, "SPEC", "auto"))); + } + if mode != "off" + && !mode.is_empty() + && mode != "ngram" + && env.help.contains("--spec-draft-n-max") + { + let cur = first_non_empty(&KNOB_SPEC_N.get(cfg), "3"); + for v in ["2", "3", "4"] { + if v != cur { + out.push(cand( + "opt-in", + format!("SPEC_N_MAX={v}"), + KNOB_SPEC_N.set(cfg, v), + )); + } + } + } + if !get(cfg, "CUDA_GRAPH_OPT").trim().eq_ignore_ascii_case("on") + && env + .user_env + .get("GGML_CUDA_GRAPH_OPT") + .is_none_or(|v| v.is_empty()) + { + out.push(cand( + "opt-in", + "CUDA_GRAPH_OPT=on", + set_key(cfg, "CUDA_GRAPH_OPT", "on"), + )); + } + if env.help.contains("--backend-sampling") + && !has_any_flag(&extra, &["-bs", "--backend-sampling"]) + && !crate::serve::env_truthy( + env.arg_env + .get("LLAMA_ARG_BACKEND_SAMPLING") + .map(String::as_str) + .unwrap_or(""), + ) + && crate::serve::split::split_mode_key(cfg) != "tensor" + { + out.push(cand( + "opt-in", + "--backend-sampling", + set_key( + cfg, + "EXTRA_ARGS", + &extra_add_switch(get(cfg, "EXTRA_ARGS"), "--backend-sampling"), + ), + )); + } + if out.is_empty() { + return (vec![], "rien à essayer".into()); + } + (out, String::new()) +} + +// --------------------------------------------------------------------------- +// Garde-fous +// --------------------------------------------------------------------------- + +/// `tuneCoreArgs` : la ligne de commande sans ce que l'optimiseur a le droit +/// de régler. +pub fn core_args(args: &[String], opt_in: bool) -> Vec { + let mut out = Vec::with_capacity(args.len()); + let mut i = 0; + while i < args.len() { + let a = &args[i]; + let (name, val, has_eq) = match a.split_once('=') { + Some((n, v)) => (n.to_string(), v.to_string(), true), + None => (a.clone(), String::new(), false), + }; + let next = |i: &mut usize| -> String { + if has_eq { + return val.clone(); + } + if *i + 1 < args.len() { + *i += 1; + return args[*i].clone(); + } + String::new() + }; + match name.as_str() { + "--host" | "--port" | "--api-key" | "--slot-save-path" | "-cram" | "--cache-ram" => { + next(&mut i); + out.push(name.clone()); + out.push("*".into()); + i += 1; + continue; + } + _ => {} + } + if opt_in && (name.starts_with("--spec-") || name == "-md" || name == "--model-draft") { + next(&mut i); + i += 1; + continue; + } + let Some(f) = flag_for(&name, opt_in) else { + out.push(a.clone()); + i += 1; + continue; + }; + if f.arity == 0 { + i += 1; + continue; + } + let v = next(&mut i); + if name == "-ot" || name == "--override-tensor" { + let rest: Vec = v + .split(',') + .filter_map(|el| { + let el = el.trim(); + let pat = el.split_once('=').map_or(el, |(p, _)| p).trim(); + (!el.is_empty() && !crate::serve::moe::ot_targets_experts(pat)) + .then(|| el.to_string()) + }) + .collect(); + if !rest.is_empty() { + out.push(name.clone()); + out.push(rest.join(",")); + } + } + i += 1; + } + out +} + +/// `tuneEnvTuned` : variables du moteur que l'optimiseur règle. +pub fn env_tuned(k: &str, opt_in: bool) -> bool { + match k { + "CUDA_SCALE_LAUNCH_QUEUES" | "GGML_OP_OFFLOAD_MIN_BATCH" => true, + "GGML_CUDA_GRAPH_OPT" => opt_in, + _ => false, + } +} + +/// `tuneLaunch` : la ligne de commande composée par `loki serve` pour un +/// essai. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Launch { + #[serde(default)] + pub bin: String, + #[serde(default)] + pub args: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub probe: Option, +} + +/// `tuneProbe` : ce que `loki serve` a sondé pour composer la ligne. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Probe { + #[serde(default)] + pub help: String, + #[serde(default)] + pub gpus: i64, + #[serde(default)] + pub vram_mib: i64, + #[serde(default)] + pub model_bytes: i64, + #[serde(default)] + pub moe: bool, + #[serde(default)] + pub cpu_threads: i64, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub arg_env: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub user_env: HashMap, + #[serde(default)] + pub queues_env: bool, +} + +/// `tuneDenature` : "" = même calcul ; sinon la première différence. +pub fn denature(base: &Launch, cand: &Launch, opt_in: bool) -> String { + let (ba, ca) = (core_args(&base.args, opt_in), core_args(&cand.args, opt_in)); + for i in 0..ba.len().max(ca.len()) { + let x = ba.get(i).map(String::as_str).unwrap_or(""); + let y = ca.get(i).map(String::as_str).unwrap_or(""); + if x != y { + return format!( + "ligne de commande différente hors réglages permis (« {y} » au lieu de « {x} »)" + ); + } + } + // Le Go parcourt une table : l'ordre des clés y est aléatoire. Ici, trié. + let keys: BTreeSet<&String> = base.env.keys().chain(cand.env.keys()).collect(); + for k in keys { + let (b, c) = ( + base.env.get(k).map(String::as_str).unwrap_or(""), + cand.env.get(k).map(String::as_str).unwrap_or(""), + ); + if !env_tuned(k, opt_in) && b != c { + return format!("variable {k} différente (« {c} » au lieu de « {b} »)"); + } + } + String::new() +} + +/// `tuneLaunchKey` : identité d'une ligne de lancement, adresse et port mis +/// à part. +pub fn launch_key(l: &Launch) -> String { + let mut full = Vec::with_capacity(l.args.len()); + let mut i = 0; + while i < l.args.len() { + match l.args[i].as_str() { + "--host" | "--port" | "--api-key" | "--slot-save-path" => { + i += 2; + continue; + } + _ => full.push(l.args[i].clone()), + } + i += 1; + } + let keys: Vec = l.env.iter().map(|(k, v)| format!("{k}={v}")).collect(); + format!("{}\x01{}", full.join("\x00"), keys.join("\x00")) +} + +fn all_keys<'a>(a: &'a Cfg, b: &'a Cfg) -> BTreeSet<&'a String> { + a.keys().chain(b.keys()).collect() +} + +/// `tuneCfgCheck` : la configuration d'un essai ne diffère de la référence +/// que par des clés permises, et son EXTRA_ARGS que par la liste blanche. +pub fn cfg_check(base: &Cfg, cand: &Cfg, opt_in: bool) -> Result<(), String> { + for k in all_keys(base, cand) { + if get(base, k) == get(cand, k) { + continue; + } + if k == "EXTRA_ARGS" { + let a = core_args(&split_args(get(base, k)), opt_in); + let b = core_args(&split_args(get(cand, k)), opt_in); + if a != b { + return Err("EXTRA_ARGS modifié hors liste blanche".into()); + } + continue; + } + if !is_key(k) && !(opt_in && is_opt_in_key(k)) { + return Err(format!("clé {k} intouchable")); + } + } + Ok(()) +} + +/// `tuneDiff` : ce qui change d'une configuration à l'autre, clé par clé. +pub fn diff(base: &Cfg, cand: &Cfg) -> Vec { + let show = |v: &str| { + if v.is_empty() { + "(absent)".to_string() + } else { + v.to_string() + } + }; + all_keys(base, cand) + .into_iter() + .filter(|k| get(base, k) != get(cand, k)) + .map(|k| format!("{k} : {} → {}", show(get(base, k)), show(get(cand, k)))) + .collect() +} + +/// `tunePatchPreset` : le preset réécrit clé par clé, vérifié. +pub fn patch_preset(content: &str, base: &Cfg, win: &Cfg) -> Result { + let changed: Vec<&String> = all_keys(base, win) + .into_iter() + .filter(|k| get(base, k) != get(win, k)) + .collect(); + let mut want = crate::presets::parse_env(content); + let mut out = content.to_string(); + for k in changed { + if !is_key(k) && !is_opt_in_key(k) { + return Err(format!("clé {k} intouchable")); + } + out = crate::presets_api::preset_set_key(&out, k, get(win, k)); + if get(win, k).is_empty() { + want.remove(k); + } else { + want.insert(k.clone(), get(win, k).to_string()); + } + } + let got = crate::presets::parse_env(&out); + if got.len() != want.len() { + return Err("réécriture du preset incohérente".into()); + } + let mut keys: Vec<&String> = want.keys().collect(); + keys.sort(); + for k in keys { + if got.get(k) != want.get(k) { + return Err(format!("réécriture du preset incohérente sur {k}")); + } + } + Ok(out) +} + +// --------------------------------------------------------------------------- +// Journal de l'essai +// --------------------------------------------------------------------------- + +/// `tuneLogFacts` : couches déportées (dernière mention ; -1 = absente) et +/// types du cache KV vus dans le journal. +pub fn log_facts(log: &str) -> (i64, i64, String) { + let off_re = regex::Regex::new(r"offloaded (\d+)/(\d+) layers to GPU").expect("regex"); + let kv_re = + regex::Regex::new(r"K \(([A-Za-z0-9_]+)\):[^\n]*V \(([A-Za-z0-9_]+)\):").expect("regex"); + let (mut off, mut total) = (-1, -1); + for m in off_re.captures_iter(log) { + off = m[1].parse().unwrap_or(0); + total = m[2].parse().unwrap_or(0); + } + let kvs: BTreeSet = kv_re + .captures_iter(log) + .map(|m| format!("{}/{}", m[1].to_lowercase(), m[2].to_lowercase())) + .collect(); + (off, total, kvs.into_iter().collect::>().join(",")) +} + +/// `tuneLoadFailure` : pourquoi un essai n'a pas chargé. +pub fn load_failure(log: &str) -> String { + let l = log.to_lowercase(); + if l.contains("out of memory") + || l.contains("cudamalloc failed") + || l.contains("failed to allocate") + { + return "mémoire insuffisante (OOM)".into(); + } + if l.contains("failed to fit") { + return "--fit n'a pas su placer le modèle".into(); + } + if l.contains("failed to load model") || l.contains("error loading model") { + return "échec du chargement du modèle".into(); + } + let trimmed = log.trim(); + if let Some(last) = trimmed.split('\n').next_back() + && !last.is_empty() + { + let last = last.trim(); + // last[:160] du Go : en octets — un caractère coupé devient U+FFFD + // par octet à l'encodage JSON. + let last = if last.len() > 160 { + crate::system::go_lossy(&last.as_bytes()[..160]) + } else { + last.to_string() + }; + return format!("moteur arrêté : {last}"); + } + "moteur arrêté au chargement".into() +} + +// --------------------------------------------------------------------------- +// Mesure et score +// --------------------------------------------------------------------------- + +/// `tuneWeights` : la forme d'un tour typique. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Weights { + #[serde(default, serialize_with = "ser_go_f64")] + pub k: f64, + #[serde(default, serialize_with = "ser_go_f64")] + pub g: f64, + #[serde(default, rename = "f_cold", serialize_with = "ser_go_f64")] + pub f_cold: f64, + #[serde(default)] + pub from: String, +} + +impl Default for Weights { + fn default() -> Self { + Weights { + k: 2000.0, + g: 600.0, + f_cold: 0.05, + from: "défaut".into(), + } + } +} + +/// Un enregistrement de la télémétrie, vu de l'optimiseur. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +pub struct PerfRec { + #[serde(default)] + pub kind: String, + #[serde(default)] + pub complete: bool, + #[serde(default)] + pub total: i64, + #[serde(default)] + pub cached: Option, + #[serde(default)] + pub new: i64, + #[serde(default, rename = "gen")] + pub generated: i64, +} + +/// `perfMedian` : la médiane (moyenne des deux du milieu). +fn median(v: &mut [f64]) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let n = v.len(); + if n % 2 == 1 { + v[n / 2] + } else { + (v[n / 2 - 1] + v[n / 2]) / 2.0 + } +} + +/// `tuneWeightsFrom` : K, G et fCold des médianes de la télémétrie quand +/// elle en a assez vu ; sinon les valeurs par défaut. +pub fn weights_from(recs: &[PerfRec]) -> Weights { + let (mut news, mut gens) = (Vec::new(), Vec::new()); + let (mut cold, mut known) = (0, 0); + for r in recs { + if !r.complete || matches!(r.kind.as_str(), "bench" | "prewarm" | "foreign") { + continue; + } + if r.new > 0 { + news.push(r.new as f64); + } + if r.generated > 0 { + gens.push(r.generated as f64); + } + if let Some(c) = r.cached + && r.total >= 2048 + { + known += 1; + if c * 10 < r.total { + cold += 1; + } + } + } + if news.len() < 20 || gens.len() < 20 { + return Weights::default(); + } + let mut w = Weights { + k: median(&mut news).clamp(128.0, 32768.0), + g: median(&mut gens).clamp(16.0, 8192.0), + from: "télémétrie".into(), + ..Default::default() + }; + if known >= 20 { + w.f_cold = cold as f64 / known as f64; + } + w +} + +/// `tuneRun` : une mesure d'un essai. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Run { + #[serde(default, serialize_with = "ser_go_f64")] + pub turn_sec: f64, + #[serde(default, skip_serializing_if = "zero", serialize_with = "ser_go_f64")] + pub cold_pp: f64, + #[serde(default, skip_serializing_if = "zero", serialize_with = "ser_go_f64")] + pub cached_pp: f64, + #[serde(default, skip_serializing_if = "zero", serialize_with = "ser_go_f64")] + pub tg: f64, + #[serde(default, serialize_with = "ser_go_f64")] + pub prose_pp: f64, + #[serde(default, serialize_with = "ser_go_f64")] + pub prose_tg: f64, + #[serde(default, skip_serializing_if = "zero_i")] + pub depth: i64, +} + +fn zero(v: &f64) -> bool { + *v == 0.0 +} + +fn zero_i(v: &i64) -> bool { + *v == 0 +} + +/// `tuneRunFrom` : un bench complet réduit à son score. +pub fn run_from(res: Option<&BenchResult>, w: &Weights) -> Result { + let Some(res) = res.filter(|r| r.predicted_per_second > 0.0 && r.prompt_per_second > 0.0) + else { + return Err("mesure vide".into()); + }; + let mut r = Run { + prose_pp: res.prompt_per_second, + prose_tg: res.predicted_per_second, + ..Default::default() + }; + let d = match &res.depth { + Some(d) if d.skipped.is_empty() => d, + _ => { + r.turn_sec = w.k / r.prose_pp + w.g / r.prose_tg; + return Ok(r); + } + }; + if !d.partial.is_empty() { + return Err(format!("mesure partielle : {}", d.partial)); + } + let Some(cold) = d.cold.as_ref().filter(|c| c.prompt_per_second > 0.0) else { + return Err("mesure en profondeur incomplète".into()); + }; + if d.turns.is_empty() || d.decode_per_second <= 0.0 { + return Err("mesure en profondeur incomplète".into()); + } + let (mut fresh, mut ms) = (0i64, 0f64); + for t in &d.turns { + if t.cached < 0 || t.expected <= 0 { + fresh = 0; + ms = 0.0; + break; + } + fresh += t.new + t.cached - t.expected; + ms += t.prompt_ms; + } + r.cached_pp = d.cached_per_second; + if fresh > 0 && ms > 0.0 { + r.cached_pp = fresh as f64 / (ms / 1000.0); + } + if r.cached_pp <= 0.0 { + return Err("prefill des tours illisible".into()); + } + r.cold_pp = cold.prompt_per_second; + r.tg = d.decode_per_second; + r.depth = d.target; + r.turn_sec = w.k / r.cached_pp + w.g / r.tg + w.f_cold * d.target as f64 / r.cold_pp; + Ok(r) +} + +/// `tuneMean`. +pub fn mean(v: &[f64]) -> f64 { + if v.is_empty() { + return 0.0; + } + v.iter().sum::() / v.len() as f64 +} + +/// `tuneSpread` : l'écart relatif entre passages ((max-min)/moyenne). +pub fn spread(v: &[f64]) -> f64 { + if v.len() < 2 { + return 0.0; + } + let lo = v.iter().copied().fold(f64::INFINITY, f64::min); + let hi = v.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let m = mean(v); + if m <= 0.0 { + return 0.0; + } + (hi - lo) / m +} + +/// `tuneMinGain`. +pub const MIN_GAIN: f64 = 0.03; + +/// `tuneWins` : le candidat bat-il la référence, d'au moins max(3 %, l'écart +/// entre passages), et sur deux passages ? +pub fn wins(base: &[f64], cand: &[f64]) -> (f64, f64, bool) { + let mb = mean(base); + if mb <= 0.0 || cand.is_empty() { + return (0.0, MIN_GAIN, false); + } + let gain = (mb - mean(cand)) / mb; + let need = MIN_GAIN.max(spread(base)).max(spread(cand)); + (gain, need, cand.len() >= 2 && gain > need) +} + +/// `tuneWorthRepeat`. +pub fn worth_repeat(base: &[f64], first: f64) -> bool { + let mb = mean(base); + mb > 0.0 && (mb - first) / mb > MIN_GAIN +} + +/// `tuneTightMiB`. +pub const TIGHT_MIB: i64 = 768; +/// `tuneVisionReserveMiB`. +pub const VISION_RESERVE_MIB: i64 = 512; + +/// `tuneHeadroom` : VRAM libre minimale sur les cartes visibles (Mio) ; +/// `(total, used)` par carte ; `visible` vide = toutes. +pub fn headroom(gpus: &[(i64, i64)], visible: &[i64]) -> Option { + let mut min_free: Option = None; + for (i, (total, used)) in gpus.iter().enumerate() { + if !visible.is_empty() && !visible.contains(&(i as i64)) { + continue; + } + if *total <= 0 { + continue; + } + let free = total - used; + min_free = Some(min_free.map_or(free, |m| m.min(free))); + } + min_free +} + +/// `tuneVisibleGPUs` : index des cartes de CUDA_VISIBLE_DEVICES ; vide = +/// toutes, ou des UUID qu'on ne sait pas relier. +pub fn visible_gpus(cfg: &Cfg) -> Vec { + let mut out = Vec::new(); + for p in get(cfg, "CUDA_VISIBLE_DEVICES").split(',') { + let p = p.trim(); + let digits = p.strip_prefix(['+', '-']).unwrap_or(p); + if digits.is_empty() || !digits.bytes().all(|c| c.is_ascii_digit()) { + return Vec::new(); + } + match p.parse() { + Ok(n) => out.push(n), + Err(_) => return Vec::new(), + } + } + out +} + +/// `tuneForceAddr` : toutes les occurrences de --host / --port pointent sur +/// l'adresse de l'essai. +pub fn force_addr(args: &[String], host: &str, port: &str) -> Vec { + let mut out = args.to_vec(); + let mut i = 0; + while i < out.len() { + let (name, has_eq) = match out[i].split_once('=') { + Some((n, _)) => (n.to_string(), true), + None => (out[i].clone(), false), + }; + let val = match name.as_str() { + "--host" => host, + "--port" => port, + _ => { + i += 1; + continue; + } + }; + if has_eq { + out[i] = format!("{name}={val}"); + } else if i + 1 < out.len() { + out[i + 1] = val.to_string(); + i += 1; + } + i += 1; + } + out +} + +// --------------------------------------------------------------------------- +// Fichiers de l'essai +// --------------------------------------------------------------------------- + +/// `writeTuneTrialConfig` (le texte) : relue, la configuration doit revenir +/// à l'identique — une valeur que le format ne sait pas représenter serait +/// un autre essai que celui voulu. +pub fn trial_config_text(cfg: &Cfg) -> Result { + let sorted: BTreeMap = + cfg.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let text = crate::presets_api::format_env(&sorted); + let back = crate::presets::parse_env(&text); + if back.len() != cfg.len() { + return Err("configuration d'essai non représentable".into()); + } + for (k, v) in &sorted { + if back.get(k) != Some(v) { + return Err(format!( + "valeur de {k} non représentable dans un fichier de configuration" + )); + } + } + Ok(text) +} diff --git a/rust/loki-rs/src/tune/job.rs b/rust/loki-rs/src/tune/job.rs new file mode 100644 index 0000000..df7bd36 --- /dev/null +++ b/rust/loki-rs/src/tune/job.rs @@ -0,0 +1,2208 @@ +//! L'optimiseur côté web — port de `backend_tune_job.go` et +//! `backend_tune_run.go` (la partie pure est dans [`super::core`]). +//! +//! ```text +//! POST /api/tune lance (202), 409 si le moteur est occupé +//! GET /api/tune/status phase, essais, ETA ; puis le résultat +//! POST /api/tune/cancel annule (l'essai en cours est arrêté, le moteur relancé) +//! POST /api/tune/apply {target:"copy"|"preset"} — SEULEMENT sur un clic +//! GET /api/tune/last dernier résultat d'un preset, et si le moteur a changé depuis +//! ``` +//! +//! Le déroulé : refus d'entrée, verrou exclusif ([`super::lock`]), vrai +//! moteur arrêté ; chaque essai est une COPIE de la configuration avec ses +//! surcharges (`LOKI_HOME/tune/run//config.env`, jamais config.env), +//! composée par « loki serve » du Go à blanc, puis lancée sur 127.0.0.1 et un +//! port libre, et mesurée par le bench complet à une profondeur fixe. Le +//! groupe de processus de l'essai est tué en sortie, quoi qu'il arrive. À la +//! fin, le vrai moteur est relancé et le verrou rendu. RIEN n'est écrit dans +//! un preset sans un geste de l'utilisateur. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use http_body_util::{BodyExt, Limited}; +use hyper::body::Incoming; +use hyper::{Method, Request, Response, StatusCode}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::value::RawValue; +use serde_json::{Map, Value, json}; +use tokio::sync::{Notify, watch}; + +use super::core::{self as tc, Cfg, Launch, Run, Weights}; +use super::lock::{self, ERR_TUNE_BUSY, Held, LockFile, Owner, Proc}; +use crate::api::Ctx; +use crate::auth::json as send_json; +use crate::bench::{self, BenchResult, Corpora, Engine, ser_go_f64}; +use crate::chat::lease::{EngineLease, LEASE_TTL, LeaseError, LeaseSignal}; +use crate::chat::wire::{EffortFallbacks, log_effort_fallback}; +use crate::gojson::{self, Kind}; +use crate::proxy::Body; +use crate::store::BK_STATE; + +/// `tuneDefaultBudget`, `tuneMaxBudget`. +pub const DEFAULT_BUDGET: Duration = Duration::from_secs(30 * 60); +pub const MAX_BUDGET: Duration = Duration::from_secs(6 * 3600); +/// `tuneLoadTimeout` : chargement d'un essai (un gros MoE relu d'un disque +/// lent prend plusieurs minutes). +pub const LOAD_TIMEOUT: Duration = Duration::from_secs(20 * 60); + +const RESULTS_KEY: &str = "tune_results"; +const APPLY_COPY: &str = "copy"; +const APPLY_PRESET: &str = "preset"; + +const TUNE_REQ: &str = r#"struct { Placement bool "json:\"placement\""; OptIn bool "json:\"optin\""; BudgetMin int "json:\"budget_min\""; Stages []string "json:\"stages\"" }"#; +const APPLY_REQ: &str = r#"struct { Target string "json:\"target\"" }"#; + +// --------------------------------------------------------------------------- +// Ce qui se montre +// --------------------------------------------------------------------------- + +/// `tuneOpts` : ce que l'utilisateur a demandé. +#[derive(Clone, Debug, Default)] +pub struct Opts { + pub placement: bool, + pub opt_in: bool, + /// Zéro : [`DEFAULT_BUDGET`]. + pub budget: Duration, + pub via: String, + /// Étapes de base à faire ; vide = toutes. + pub stages: Vec, +} + +fn zero(v: &f64) -> bool { + *v == 0.0 +} + +/// Une liste vide s'écrit `null`, comme une tranche nil du Go. +fn ser_nil_vec(v: &[T], s: S) -> Result { + if v.is_empty() { + s.serialize_none() + } else { + v.serialize(s) + } +} + +fn de_nil_vec<'de, T: Deserialize<'de>, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + Ok(Option::>::deserialize(d)?.unwrap_or_default()) +} + +/// `tuneTrial` : un essai, tel que montré à l'utilisateur. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Trial { + #[serde(default)] + pub stage: String, + #[serde(default)] + pub label: String, + /// en cours, mesuré, retenu, écarté, échec, doublon + #[serde(default)] + pub status: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub why: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub runs: Vec, + #[serde(default, skip_serializing_if = "zero", serialize_with = "ser_go_f64")] + pub turn_sec: f64, + #[serde(default, serialize_with = "ser_go_f64")] + pub gain: f64, + /// VRAM libre minimale après le bench ; -1 = non mesurée. + #[serde(default, rename = "headroom_mib")] + pub headroom: i64, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub offloaded: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changes: Vec, + #[serde(default, serialize_with = "ser_go_f64")] + pub sec: f64, +} + +/// `tuneResult` : l'issue d'une optimisation. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct TuneResult { + #[serde(default)] + pub preset_id: String, + #[serde(default)] + pub preset_name: String, + #[serde(default)] + pub base_fp: String, + #[serde(default)] + pub preset_fp: String, + #[serde(default)] + pub engine: String, + #[serde(default)] + pub engine_build: i64, + #[serde(default)] + pub weights: Weights, + #[serde(default)] + pub depth: i64, + #[serde(default)] + pub n_ctx: i64, + #[serde(default)] + pub slots: i64, + #[serde(default)] + pub baseline: Trial, + #[serde( + default, + serialize_with = "ser_nil_vec", + deserialize_with = "de_nil_vec" + )] + pub trials: Vec, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub best: String, + #[serde(default, serialize_with = "ser_go_f64")] + pub gain: f64, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub set: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub old: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changes: Vec, + #[serde(default)] + pub placement: bool, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub partial: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub skipped: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, + #[serde(default)] + pub opt_in: bool, + #[serde(default)] + pub at: i64, +} + +/// `tuneTracker` : la progression, lue par l'interface. +pub struct Tracker { + inner: Mutex, +} + +struct TrackerState { + phase: String, + started: Instant, + /// Fin de la mesure ; `None` = en cours. + ended: Option, + trials: Vec, + avg_sec: f64, + left: i64, + budget: Duration, +} + +impl Tracker { + fn new() -> Self { + Tracker { + inner: Mutex::new(TrackerState { + phase: String::new(), + started: Instant::now(), + ended: None, + trials: Vec::new(), + avg_sec: 0.0, + left: 0, + budget: Duration::ZERO, + }), + } + } + + fn lock(&self) -> MutexGuard<'_, TrackerState> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn set_phase(&self, phase: impl Into) { + self.lock().phase = phase.into(); + } + + /// `put` : `i` hors des bornes ajoute ; rend l'index. + fn put(&self, i: Option, t: Trial) -> usize { + let mut st = self.lock(); + match i { + Some(i) if i < st.trials.len() => { + st.trials[i] = t; + i + } + _ => { + st.trials.push(t); + st.trials.len() - 1 + } + } + } + + fn snapshot(&self) -> Map { + let st = self.lock(); + let end = st.ended.unwrap_or_else(Instant::now); + let el = end.saturating_duration_since(st.started); + let mut eta = -1.0; + if st.avg_sec > 0.0 { + let rest = st.budget.as_secs_f64() - el.as_secs_f64(); + eta = (st.avg_sec * st.left as f64).min(rest).max(0.0); + } + let mut m = Map::new(); + m.insert("phase".into(), json!(st.phase)); + m.insert("elapsed_sec".into(), go_f64(el.as_secs_f64())); + m.insert("eta_sec".into(), go_f64(eta)); + m.insert( + "trials".into(), + if st.trials.is_empty() { + Value::Null + } else { + serde_json::to_value(&st.trials).unwrap_or(Value::Null) + }, + ); + m + } +} + +/// Un flottant écrit comme le Go (entier sans « .0 »). +fn go_f64(v: f64) -> Value { + if v.is_finite() && v.fract() == 0.0 && v.abs() < 1e15 { + json!(v as i64) + } else { + json!(v) + } +} + +// --------------------------------------------------------------------------- +// Le job +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct State { + running: bool, + /// Entre le contrôle et la prise du bail : un second lancement est refusé. + starting: bool, + tracker: Option>, + cancel: Option>, + result: Option, + err: String, + applying: bool, + apply_log: Vec, + apply_msg: String, + apply_err: String, +} + +/// L'optimiseur de la façade. +pub struct Tune { + home: PathBuf, + /// Le « loki » du Go, lancé en `serve` pour chaque essai. + go_bin: PathBuf, + pub lock: LockFile, + state: Mutex, + effort: EffortFallbacks, + client: reqwest::Client, + builds: Mutex>, + /// Chargement d'un essai, et attente du moteur après une application. + pub load_timeout: Duration, + /// Entre deux sondes de /health d'un essai (1 s). + pub poll: Duration, + /// Entre deux sondes du moteur après une application (2 s). + pub probe_poll: Duration, + /// Sans nvidia-smi, la pause qui laisse la mémoire revenir après un essai + /// (2 s). + pub settle: Duration, + /// Variables ajoutées aux essais (les tests). + pub trial_env: Vec<(String, String)>, + /// Durée du bail du moteur (renouvelé au tiers). + pub lease_ttl: Duration, +} + +impl Tune { + pub fn new(home: PathBuf, go_bin: PathBuf) -> Self { + let lock = LockFile::new(&home); + Self::with_lock(home, go_bin, lock) + } + + pub fn with_lock(home: PathBuf, go_bin: PathBuf, lock: LockFile) -> Self { + crate::chat::engine::install_crypto(); + Tune { + home, + go_bin, + lock, + state: Mutex::new(State::default()), + effort: EffortFallbacks::new(), + client: reqwest::Client::builder().build().unwrap_or_default(), + builds: Mutex::new(HashMap::new()), + load_timeout: LOAD_TIMEOUT, + poll: Duration::from_secs(1), + probe_poll: Duration::from_secs(2), + settle: Duration::from_secs(2), + trial_env: Vec::new(), + lease_ttl: LEASE_TTL, + } + } + + fn st(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn effort<'a>(&'a self, ctx: &'a Ctx) -> &'a EffortFallbacks { + match &ctx.chat { + Some(c) => &c.shared.effort, + None => &self.effort, + } + } + + /// `tuneStart` : le statut HTTP et la réponse. + pub async fn start(self: &Arc, ctx: &Ctx, mut opts: Opts) -> (StatusCode, Value) { + let fail = |code: StatusCode, e: &str| (code, json!({ "ok": false, "error": e })); + if let Err(e) = self.preflight(ctx).await { + let code = if crate::engstatus::is_external(&ctx.store.config().await) { + StatusCode::BAD_REQUEST + } else { + StatusCode::CONFLICT + }; + return fail(code, &e); + } + { + let mut st = self.st(); + if st.running || st.applying || st.starting { + return fail(StatusCode::CONFLICT, ERR_TUNE_BUSY); + } + st.starting = true; + } + // Le verrou de la discussion : un message, une tâche, une compaction + // reçoivent « optimisation en cours » ; le stop du chat annule. + let lease = match EngineLease::acquire_for(ctx.store.clone(), self.lease_ttl, "tune").await + { + Ok(l) => l, + Err(e) => { + self.st().starting = false; + let msg = match e { + LeaseError::Busy { error, .. } => error, + LeaseError::Other(e) => e, + }; + return fail(StatusCode::CONFLICT, &msg); + } + }; + let (tx, rx) = watch::channel(false); + let tracker = Arc::new(Tracker::new()); + { + let mut st = self.st(); + st.starting = false; + st.running = true; + st.tracker = Some(tracker.clone()); + st.cancel = Some(tx); + st.result = None; + st.err.clear(); + st.apply_log.clear(); + st.apply_msg.clear(); + st.apply_err.clear(); + } + opts.via = "web".into(); + let (me, ctx2) = (self.clone(), ctx.clone()); + tokio::spawn(async move { me.job(ctx2, opts, tracker, rx, lease).await }); + (StatusCode::ACCEPTED, json!({ "ok": true })) + } + + async fn job( + self: Arc, + ctx: Ctx, + opts: Opts, + tracker: Arc, + rx: watch::Receiver, + mut lease: EngineLease, + ) { + let run = self.run_tune(&ctx, opts, &tracker, rx.clone()); + tokio::pin!(run); + let mut interrupted = false; + let res = loop { + tokio::select! { + r = &mut run => break r, + s = lease.interrupted(), if !interrupted => { + interrupted = true; + if matches!(s, LeaseSignal::Stop | LeaseSignal::Lost) + && let Some(c) = &self.st().cancel + { + let _ = c.send(true); + } + } + } + }; + tracker.lock().ended = Some(Instant::now()); + lease.release().await; + let canceled = *rx.borrow(); + let mut st = self.st(); + st.running = false; + st.cancel = None; + match res { + Ok(r) => st.result = Some(r), + Err(e) => { + st.result = None; + st.err = if canceled { + "optimisation annulée".into() + } else { + e + }; + } + } + } + + /// `tuneStatus`. + pub fn status(&self) -> Value { + let st = self.st(); + let mut m = Map::new(); + m.insert("ok".into(), json!(true)); + m.insert("running".into(), json!(st.running)); + m.insert("applying".into(), json!(st.applying)); + if let Some(t) = &st.tracker { + m.extend(t.snapshot()); + } + if !st.running { + if let Some(r) = &st.result { + m.insert( + "result".into(), + serde_json::to_value(r).unwrap_or(Value::Null), + ); + } + if !st.err.is_empty() { + m.insert("error".into(), json!(st.err)); + } + } + if !st.apply_log.is_empty() || !st.apply_msg.is_empty() || !st.apply_err.is_empty() { + let log = if st.apply_log.is_empty() { + Value::Null + } else { + json!(st.apply_log) + }; + m.insert( + "apply".into(), + json!({ "log": log, "message": st.apply_msg, "error": st.apply_err }), + ); + } + // Une optimisation lancée ailleurs (« loki tune » en ligne de commande). + if !st.running + && !st.applying + && let Some(o) = self.lock.active() + { + m.insert("elsewhere".into(), json!(o.via)); + } + Value::Object(m) + } + + /// `handleTuneCancel`. + pub fn cancel(&self) -> bool { + let st = self.st(); + match (&st.cancel, st.running) { + (Some(c), true) => { + let _ = c.send(true); + true + } + _ => false, + } + } + + // ----------------------------------------------------------------------- + // Ce que le Go prête + // ----------------------------------------------------------------------- + + /// `/internal/v1/tune/context` : refus du processus web, moteur actif, + /// télémétrie. + async fn go_context(ctx: &Ctx) -> Value { + ctx.store + .internal_json(Method::GET, "/internal/v1/tune/context", None) + .await + .unwrap_or(Value::Null) + } + + /// `engineNeedsStop` : le moteur de la façade si elle le pilote, sinon le + /// Go. + async fn main_active(ctx: &Ctx) -> bool { + if ctx.engine_ctl.is_some() { + return crate::enginectl::service_is_active(ctx); + } + Self::go_context(ctx).await["main_active"].as_bool() == Some(true) + } + + /// `serviceActionOS` : démarrage ou arrêt SANS le refus de l'optimiseur — + /// c'est nous qui tenons le verrou. + async fn svc_os(ctx: &Ctx, action: &str) -> Result<(), String> { + if let Some(ctl) = &ctx.engine_ctl { + return ctl.run(action).await; + } + let (code, v) = ctx + .store + .internal_post( + "/internal/v1/service/action", + &json!({ "action": action, "os": true }), + ) + .await?; + if !code.is_success() { + return Err(v + .get("error") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or_else(|| format!("service {action} : {code}"))); + } + match v.get("error").and_then(Value::as_str) { + Some(e) if !e.is_empty() => Err(e.to_string()), + _ => Ok(()), + } + } + + /// `preflightEngine`. + async fn preflight_engine(ctx: &Ctx) -> Result<(), String> { + let cfg = ctx.store.config().await; + let d = crate::models::model_dirs(ctx).await; + tokio::task::spawn_blocking(move || { + crate::enginectl::preflight_engine(&d.home, &cfg, &d.stored, d.env.as_deref()) + }) + .await + .unwrap_or_else(|e| Err(e.to_string())) + } + + /// La clé du moteur local (`effectiveAPIKey`), pour les essais comme + /// pour le vrai moteur. + async fn engine_key(ctx: &Ctx) -> String { + ctx.store + .internal_json( + Method::GET, + "/internal/v1/oai/route?method=POST&path=%2Fv1%2Fchat%2Fcompletions", + None, + ) + .await + .ok() + .and_then(|v| v.get("key").and_then(Value::as_str).map(str::to_string)) + .unwrap_or_default() + } + + fn engine(&self, base: String, key: String, cancel: watch::Receiver) -> Engine { + Engine { + base, + key, + client: self.client.clone(), + cancel, + req_timeout: bench::REQ_TIMEOUT, + served: Arc::new(AtomicU64::new(0)), + } + } + + // ----------------------------------------------------------------------- + // Refus d'entrée + // ----------------------------------------------------------------------- + + /// `tuneActivePreset` : le preset dont l'empreinte est celle de la + /// configuration active. + async fn active_preset( + &self, + ctx: &Ctx, + cfg: &Cfg, + ) -> Result<(String, String, String), String> { + let order = crate::presets::parse_preset_order( + ctx.store + .get(BK_STATE, crate::presets::PRESET_ORDER_KEY) + .await + .ok() + .flatten() + .as_deref(), + ); + let list = crate::presets::list_presets(&self.home, cfg, &order)?; + match list.into_iter().find(|p| p.active) { + Some(p) => { + let content = crate::presets_api::read_preset(&self.home, &p.id)?; + Ok((p.id, p.name, content)) + } + None => Err("aucun preset ne correspond à la configuration active : enregistre-la comme preset (ou bascule sur l'un d'eux) avant d'optimiser".into()), + } + } + + /// `tunePreflight` (processus web). + async fn preflight(&self, ctx: &Ctx) -> Result<(), String> { + let cfg = ctx.store.config().await; + if crate::engstatus::is_external(&cfg) { + return Err("optimisation indisponible : le preset actif est une API externe".into()); + } + if crate::strata::is_strata(&cfg) { + return Err("optimisation indisponible : le preset actif tourne sur Strata, réglé par sa propre fenêtre".into()); + } + if self.lock.active().is_some() { + return Err(ERR_TUNE_BUSY.into()); + } + self.active_preset(ctx, &cfg).await?; + let gc = Self::go_context(ctx).await; + if let Some(b) = gc["busy"].as_str().filter(|b| !b.is_empty()) { + return Err(b.to_string()); + } + if crate::enginectl::generating(ctx).await { + return Err("génération en cours".into()); + } + let active = match &ctx.engine_ctl { + Some(_) => crate::enginectl::service_is_active(ctx), + None => gc["main_active"].as_bool() == Some(true), + }; + if active { + let (_tx, rx) = watch::channel(false); + let e = self.engine( + format!("http://localhost:{}", crate::engstatus::llm_port_raw(&cfg)), + Self::engine_key(ctx).await, + rx, + ); + if e.idle().await.is_err() { + return Err("le moteur traite une requête (discussion, tâche ou client /v1) : réessaie une fois libre".into()); + } + } + Ok(()) + } + + // ----------------------------------------------------------------------- + // Le verrou : ménage d'un verrou périmé + // ----------------------------------------------------------------------- + + /// `tuneReapStale` : l'essai orphelin d'un verrou périmé est arrêté + /// (identité complète vérifiée), une application interrompue défaite. + /// Rend `MainWasActive` du verrou écarté. + async fn reap_stale(&self, ctx: &Ctx) -> bool { + let Some(o) = self.lock.reap_claim() else { + return false; + }; + if let Some(t) = &o.trial + && self.lock.proc_alive(t) + { + eprintln!( + "[loki tune] essai orphelin (PID {}) d'une optimisation interrompue : arrêt", + t.pid + ); + let pid = t.pid; + let _ = tokio::task::spawn_blocking(move || { + lock::kill_tree(pid, &|| !lock::pid_alive(pid)); + }) + .await; + } + self.undo_apply(ctx, &o).await; + o.main_was_active + } + + /// `tuneUndoApply` : une application interrompue avant sa vérification + /// remet la version d'avant du preset. + async fn undo_apply(&self, ctx: &Ctx, o: &Owner) { + if o.phase != "application" || o.backup.is_empty() || o.preset_id.is_empty() { + return; + } + let b = match std::fs::read(&o.backup) { + Ok(b) => String::from_utf8_lossy(&b).into_owned(), + Err(e) => { + eprintln!("[loki tune] application interrompue : sauvegarde illisible ({e})"); + return; + } + }; + if let Err(e) = + crate::presets_api::save_applying(ctx, &o.preset_id, &o.preset_name, &b, "apply").await + { + eprintln!( + "[loki tune] application interrompue : retour impossible ({e}) — l'ancienne version est dans {}", + o.backup + ); + return; + } + eprintln!( + "[loki tune] application interrompue avant sa vérification : ancienne version du preset rétablie" + ); + } + + async fn acquire(&self, ctx: &Ctx, via: &str, main_active: bool) -> Result, String> { + self.lock + .acquire(via, main_active, || self.reap_stale(ctx)) + .await + } + + // ----------------------------------------------------------------------- + // Le déroulé + // ----------------------------------------------------------------------- + + /// `runTune` : le verrou, l'arrêt et la relance du vrai moteur sont faits + /// ici. + pub async fn run_tune( + &self, + ctx: &Ctx, + mut opts: Opts, + tr: &Tracker, + cancel: watch::Receiver, + ) -> Result { + if opts.budget.is_zero() { + opts.budget = DEFAULT_BUDGET; + } + opts.budget = opts.budget.min(MAX_BUDGET); + { + let mut st = tr.lock(); + st.started = Instant::now(); + st.budget = opts.budget; + } + let cfg = ctx.store.config().await; + let preset = self.active_preset(ctx, &cfg).await?; + let main_active = Self::main_active(ctx).await; + let held = self.acquire(ctx, &opts.via, main_active).await?; + let mut out = self + .run_held(ctx, &opts, tr, cancel, &held, &cfg, preset) + .await; + // Le vrai moteur revient AVANT que le verrou ne soit rendu : personne + // ne le relance par-dessus un essai. + if main_active { + held.set_phase("relance"); + tr.set_phase("relance du moteur principal"); + let mut failed = |msg: String| { + eprintln!("[loki tune] {msg}"); + match &mut out { + Ok(r) => r.notes.push(format!("⚠️ {msg}")), + Err(e) => *e = format!("{e} — {msg}"), + } + }; + match Self::preflight_engine(ctx).await { + Err(e) => failed(format!( + "relance du moteur principal impossible : {}", + crate::enginectl::plain_err(&e) + )), + Ok(()) => { + if let Err(e) = Self::svc_os(ctx, "start").await { + failed(format!( + "relance du moteur principal : {}", + crate::enginectl::plain_err(&e) + )); + } + } + } + crate::sysinfo::GPU_MONITOR.invalidate().await; + } + held.release(); + out + } + + #[allow(clippy::too_many_arguments)] + async fn run_held( + &self, + ctx: &Ctx, + opts: &Opts, + tr: &Tracker, + cancel: watch::Receiver, + held: &Held<'_>, + cfg: &Cfg, + (preset_id, preset_name, content): (String, String, String), + ) -> Result { + let canceled = || *cancel.borrow(); + let run_dir = self.home.join("tune").join("run"); + let _ = std::fs::remove_dir_all(&run_dir); + mkdir_private(&run_dir).map_err(|e| e.to_string())?; + let mut t = Trialer { + tune: self, + dir: run_dir, + seq: 0, + held, + key: Self::engine_key(ctx).await, + }; + if held.snapshot().main_was_active { + tr.set_phase("arrêt du moteur principal"); + let gpus = crate::sysinfo::gpu_stats().await; + let before = crate::enginectl::gpu_used_mb(&gpus); + if let Err(e) = Self::svc_os(ctx, "stop").await { + return Err(format!( + "arrêt du moteur : {}", + crate::enginectl::plain_err(&e) + )); + } + crate::enginectl::dictate_stop(ctx).await; + if !gpus.is_empty() { + gpu_settled(before).await; + } + crate::sysinfo::GPU_MONITOR.invalidate().await; + } + let perf: Vec = + serde_json::from_value(Self::go_context(ctx).await["perf"].clone()).unwrap_or_default(); + let mut res = TuneResult { + preset_id, + preset_name, + base_fp: crate::presets::config_fingerprint(cfg), + preset_fp: crate::presets::preset_fingerprint(&content), + opt_in: opts.opt_in, + at: unix_secs(), + weights: tc::weights_from(&perf), + ..Default::default() + }; + + // Référence : la ligne composée telle quelle, la sonde de la machine. + tr.set_phase("référence : composition de la ligne de commande"); + let (base_dir, base_port) = t.prepare(cfg)?; + let base_launch = t + .dry(&cancel, &base_dir) + .await + .map_err(|e| format!("la configuration active ne se lance pas : {e}"))?; + let probe = base_launch.probe.clone().unwrap_or_default(); + let smi = crate::sysinfo::gpu_stats().await; + let mut env = tc::Env { + help: probe.help.clone(), + gpus: probe.gpus, + vram_mib: probe.vram_mib, + model_bytes: probe.model_bytes, + moe: probe.moe, + phys_cores: phys_cores(&probe), + smi: !smi.is_empty(), + arg_env: probe.arg_env.clone(), + user_env: probe.user_env.clone(), + queues_env: probe.queues_env, + placement: opts.placement, + opt_in: opts.opt_in, + }; + let visible = tc::visible_gpus(cfg); + if env.vram_mib == 0 && env.smi { + for (i, g) in smi.iter().enumerate() { + if visible.is_empty() || visible.contains(&(i as i64)) { + env.vram_mib += g.total; + } + } + } + if env.gpus == 0 && env.smi { + env.gpus = if visible.is_empty() { + smi.len() as i64 + } else { + visible.len() as i64 + }; + } + let extra = crate::models::split_args(cfg.get("EXTRA_ARGS").map_or("", String::as_str)); + let (kt, vt, _) = crate::serve::fidelity::effective_kv_types(cfg, &extra, &env.arg_env); + let (label, quant) = bench::kv_label(&kt, &vt); + if quant { + res.notes.push(format!( + "cache KV {label} quantifié : zone grise, conservé tel quel (jamais modifié par l'optimiseur)" + )); + } + if !env.smi { + res.notes.push("VRAM libre non mesurable (nvidia-smi absent) : ni micro-lot plus grand ni marges --fit essayés".into()); + } + let (_, total) = crate::sysinfo::ram_usage_mb(); + if total > 0 && env.model_bytes > total << 20 { + res.notes.push("modèle plus gros que la RAM : chaque essai le relit depuis le disque — prévoir un budget plus long, ou ne garder que quelques étapes".into()); + } + let mut reserve = tc::TIGHT_MIB; + if !cfg.get("MMPROJ").map_or("", |s| s.trim()).is_empty() { + reserve += tc::VISION_RESERVE_MIB; + } + let cpu_placed = !crate::serve::queues::tensor_override(&extra, &env.arg_env).is_empty(); + let hybrid = crate::bench::job::model_hybrid(ctx, cfg).await; + + // La référence se mesure deux fois : son écart entre passages est le + // bruit que tout gain doit dépasser. + tr.set_phase("référence : chargement"); + let mut bt = Trial { + stage: "référence".into(), + label: "configuration actuelle".into(), + status: "en cours".into(), + headroom: -1, + ..Default::default() + }; + let bi = tr.put(None, bt.clone()); + let t0 = Instant::now(); + let (base, mut stop) = t.load(&cancel, &base_dir, base_port, &base_launch).await; + let base = match base { + Ok(b) => b, + Err(e) => { + stop.stop().await; + return Err(format!("la référence ne charge pas : {e}")); + } + }; + res.engine = base.build.clone(); + res.n_ctx = base.n_ctx; + res.slots = base.slots; + res.engine_build = build_number(&base.build); + // Une profondeur pour tous : celle que la référence peut tenir avec le + // plus gros préchauffage essayé (2 × 4096 jetons, au plus un quart du + // contexte). + let warm_max = (2 * 4096).min(base.n_ctx / 4).max(64) + bench::WARM_GEN; + res.depth = bench::depth_for(base.n_ctx, cpu_placed, warm_max).0; + let m = Measure { + tune: self, + ctx, + tr, + env: &env, + cpu_placed, + hybrid, + weights: res.weights.clone(), + bopts: bench::Opts { + mode: bench::MODE_FULL.into(), + prompt: 2000, + predict: 300, + depth: res.depth, + }, + corp: Corpora::default_set(), + }; + let headroom = || async { + if !env.smi { + return (-1, false); + } + let g: Vec<(i64, i64)> = crate::sysinfo::gpu_stats() + .await + .iter() + .map(|g| (g.total, g.used)) + .collect(); + match tc::headroom(&g, &visible) { + Some(free) => (free, free < reserve), + None => (-1, false), + } + }; + let (base_runs, err) = m.measure(cfg, &base, 2, "référence").await; + let (hr, _) = headroom().await; + stop.stop().await; + if let Some(e) = err { + return Err(format!("mesure de la référence : {e}")); + } + bt.status = "mesuré".into(); + bt.turn_sec = turn_mean(&base_runs); + bt.runs = base_runs.clone(); + bt.headroom = hr; + bt.sec = t0.elapsed().as_secs_f64(); + if base.offloaded >= 0 { + bt.offloaded = format!("{}/{}", base.offloaded, base.layers); + } + res.baseline = bt.clone(); + tr.put(Some(bi), bt.clone()); + if hr >= 0 && hr < reserve { + res.notes.push(format!( + "la configuration actuelle laisse déjà moins de {reserve} Mio libres sur une carte" + )); + } + + let (mut best, mut best_runs, mut best_label) = + (cfg.clone(), base_runs.clone(), String::new()); + let mut durations = vec![bt.sec]; + let mut measured: HashMap = + HashMap::from([(tc::launch_key(&base_launch), bt.label.clone())]); + let started = tr.lock().started; + let deadline = started + opts.budget; + + 'stages: for (si, (name, stage_gen)) in tc::STAGES.iter().enumerate() { + if !tc::stage_wanted(name, &opts.stages) { + res.skipped.push(format!("{name} : non demandé")); + continue; + } + let (cands, why) = stage_gen(&best, &env); + if !why.is_empty() { + res.skipped.push(format!("{name} : {why}")); + continue; + } + for (ci, c) in cands.iter().enumerate() { + // Ce qui reste à faire, pour l'estimation : les essais + // restants de l'étape, plus ceux des étapes suivantes vus + // depuis la meilleure configuration du moment. + let mut left = (cands.len() - ci) as i64; + for (nx, ngen) in &tc::STAGES[si + 1..] { + if tc::stage_wanted(nx, &opts.stages) { + left += ngen(&best, &env).0.len() as i64; + } + } + let avg = tc::mean(&durations); + { + let mut st = tr.lock(); + st.avg_sec = avg; + st.left = left; + } + if canceled() { + res.partial = "annulé".into(); + break 'stages; + } + let until = deadline.saturating_duration_since(Instant::now()); + if until.as_secs_f64() < avg { + res.partial = format!( + "budget de {} atteint : {left} essai(s) non faits", + bench::go_dur(round_minute(opts.budget)) + ); + break 'stages; + } + let mut trial = Trial { + stage: c.stage.clone(), + label: c.label.clone(), + status: "en cours".into(), + headroom: -1, + changes: tc::diff(cfg, &c.cfg), + ..Default::default() + }; + let idx = tr.put(None, trial.clone()); + let mut finish = |trial: &mut Trial, status: &str, why: String| { + trial.status = status.into(); + trial.why = why; + tr.put(Some(idx), trial.clone()); + res.trials.push(trial.clone()); + }; + if let Err(e) = tc::cfg_check(cfg, &c.cfg, opts.opt_in) { + finish(&mut trial, "écarté", format!("dénature : {e}")); + continue; + } + tr.set_phase(format!("{name} — {} : composition", c.label)); + let (dir, port) = match t.prepare(&c.cfg) { + Ok(x) => x, + Err(e) => { + finish(&mut trial, "échec", e); + continue; + } + }; + let launch = match t.dry(&cancel, &dir).await { + Ok(l) => l, + Err(e) => { + if canceled() { + finish(&mut trial, "échec", "annulé".into()); + res.partial = "annulé".into(); + break 'stages; + } + finish(&mut trial, "écarté", format!("refusé au lancement : {e}")); + continue; + } + }; + let why = tc::denature(&base_launch, &launch, opts.opt_in); + if !why.is_empty() { + finish(&mut trial, "écarté", format!("dénature : {why}")); + continue; + } + let key = tc::launch_key(&launch); + if let Some(prev) = measured.get(&key) { + let prev = prev.clone(); + finish( + &mut trial, + "doublon", + format!("même ligne de commande que « {prev} »"), + ); + continue; + } + measured.insert(key, c.label.clone()); + let t1 = Instant::now(); + tr.set_phase(format!("{name} — {} : chargement", c.label)); + let (l, mut stop) = t.load(&cancel, &dir, port, &launch).await; + let l = match l { + Ok(l) => l, + Err(e) => { + stop.stop().await; + if canceled() { + finish(&mut trial, "échec", "annulé".into()); + res.partial = "annulé".into(); + break 'stages; + } + finish(&mut trial, "échec", e); + durations.push(t1.elapsed().as_secs_f64()); + continue; + } + }; + trial.sec = t1.elapsed().as_secs_f64(); + if l.offloaded >= 0 { + trial.offloaded = format!("{}/{}", l.offloaded, l.layers); + } + // Contre-vérification à l'exécution : même contexte, mêmes + // slots, mêmes types de cache que la référence. + if l.n_ctx != base.n_ctx || l.slots != base.slots { + stop.stop().await; + finish( + &mut trial, + "écarté", + format!( + "dénature : contexte {} × {} slot(s) au lieu de {} × {}", + l.n_ctx, l.slots, base.n_ctx, base.slots + ), + ); + continue; + } + if !base.kv.is_empty() && !l.kv.is_empty() && l.kv != base.kv { + stop.stop().await; + finish( + &mut trial, + "écarté", + format!("dénature : cache KV {} au lieu de {}", l.kv, base.kv), + ); + continue; + } + let fit_regressed = + base.offloaded >= 0 && l.offloaded >= 0 && l.offloaded < base.offloaded; + let phase = format!("{name} — {}", c.label); + let (mut runs, mut err) = m.measure(&c.cfg, &l, 1, &phase).await; + if err.is_none() + && runs.len() == 1 + && tc::worth_repeat(&turn_secs(&best_runs), runs[0].turn_sec) + && !fit_regressed + { + let (more, e) = m + .measure(&c.cfg, &l, 1, &format!("{phase} (confirmation)")) + .await; + runs.extend(more); + err = e; + } + let (hr, tight) = headroom().await; + stop.stop().await; + trial.sec = t1.elapsed().as_secs_f64(); + durations.push(trial.sec); + trial.runs = runs.clone(); + trial.headroom = hr; + if let Some(e) = err { + if canceled() { + finish(&mut trial, "échec", "annulé".into()); + res.partial = "annulé".into(); + break 'stages; + } + finish(&mut trial, "échec", format!("mesure : {e}")); + continue; + } + trial.turn_sec = turn_mean(&runs); + let (gain, need, wins) = tc::wins(&turn_secs(&best_runs), &turn_secs(&runs)); + trial.gain = gain; + let (prose_best, prose_cand) = (prose_mean(&best_runs), prose_mean(&runs)); + if fit_regressed { + finish( + &mut trial, + "écarté", + format!( + "fit en recul : {} couches déportées au lieu de {} (performance)", + l.offloaded, base.offloaded + ), + ); + } else if tight { + finish( + &mut trial, + "écarté", + format!("trop juste : {hr} Mio libres sur une carte (< {reserve})"), + ); + } else if prose_best > 0.0 && prose_cand < 0.95 * prose_best { + finish( + &mut trial, + "mesuré", + format!( + "decode en prose plus lent de {:.0} %", + (1.0 - prose_cand / prose_best) * 100.0 + ), + ); + } else if wins { + best = c.cfg.clone(); + best_runs = runs.clone(); + best_label = c.label.clone(); + if c.stage == "placement" { + res.placement = true; + } + finish( + &mut trial, + "retenu", + format!( + "−{:.1} % par tour (seuil {:.1} %)", + gain * 100.0, + need * 100.0 + ), + ); + } else if runs.len() < 2 && gain > 0.0 { + finish( + &mut trial, + "mesuré", + format!( + "−{:.1} %, sous le seuil de {:.0} %", + gain * 100.0, + tc::MIN_GAIN * 100.0 + ), + ); + } else { + let mut delta = format!("{:+.1} %", -gain * 100.0); + if gain * 1000.0 > -0.5 && gain * 1000.0 < 0.5 { + delta = "≈ 0 %".into(); + } + finish( + &mut trial, + "mesuré", + format!("{delta} (seuil {:.1} %)", need * 100.0), + ); + } + } + } + tr.lock().left = 0; + + res.best = best_label.clone(); + if !best_label.is_empty() { + let mb = turn_mean(&base_runs); + res.gain = (mb - turn_mean(&best_runs)) / mb; + res.changes = tc::diff(cfg, &best); + for k in cfg.keys().chain(best.keys()) { + let (a, b) = ( + cfg.get(k).map_or("", String::as_str), + best.get(k).map_or("", String::as_str), + ); + if a != b { + res.set.insert(k.clone(), b.to_string()); + res.old.insert(k.clone(), a.to_string()); + } + } + } + if canceled() && res.partial.is_empty() { + res.partial = "annulé".into(); + } + save_result(ctx, &res).await; + Ok(res) + } + + // ----------------------------------------------------------------------- + // Application + // ----------------------------------------------------------------------- + + /// `tunePlanApply` : le contenu du preset réglé, sans rien écrire. + /// Refusé si le preset a changé depuis la mesure. + pub fn plan_apply(&self, res: &TuneResult) -> Result<(String, String), String> { + if res.set.is_empty() { + return Err("rien à appliquer : la configuration actuelle reste la meilleure".into()); + } + let content = crate::presets_api::read_preset(&self.home, &res.preset_id) + .map_err(|e| format!("preset « {} » introuvable : {e}", res.preset_name))?; + if crate::presets::preset_fingerprint(&content) != res.preset_fp { + return Err("le preset a changé depuis la mesure : relance l'optimisation".into()); + } + let base = crate::presets::parse_env(&content); + let mut win = base.clone(); + for (k, v) in &res.set { + if v.is_empty() { + win.remove(k); + } else { + win.insert(k.clone(), v.clone()); + } + } + tc::cfg_check(&base, &win, res.opt_in)?; + let patched = tc::patch_preset(&content, &base, &win)?; + Ok((content, patched)) + } + + /// `tuneApply` : dans une COPIE du preset, ou dans le preset lui-même — + /// encore actif ; le moteur redémarre alors sur la nouvelle version, qui + /// doit répondre, garder son contexte et ses slots, et passer une sonde ; + /// sinon l'ancienne version est rétablie. + pub async fn apply( + &self, + ctx: &Ctx, + res: &TuneResult, + target: &str, + say: &(dyn Fn(String) + Send + Sync), + ) -> Result { + let (content, patched) = self.plan_apply(res)?; + match target { + APPLY_COPY => { + let name = format!("{} (optimisé)", res.preset_name); + let id = crate::presets_api::save_preset(&self.home, "", &name, &patched)?; + return Ok(format!( + "copie « {name} » créée ({id}) — bascule dessus pour l'utiliser" + )); + } + APPLY_PRESET => {} + t => return Err(format!("cible inconnue : {t}")), + } + let cur = ctx.store.config().await; + if crate::engstatus::is_external(&cur) { + return Err("le preset actif est une API externe".into()); + } + if crate::presets::preset_fingerprint(&content) != crate::presets::config_fingerprint(&cur) + { + return Err("ce preset n'est plus celui en service : bascule dessus, ou enregistre le résultat dans une copie".into()); + } + if crate::presets::config_fingerprint(&cur) != res.base_fp { + return Err( + "la configuration active a changé depuis la mesure : relance l'optimisation".into(), + ); + } + let held = self + .acquire(ctx, "apply", Self::main_active(ctx).await) + .await?; + held.set_phase("application"); + let out = self + .apply_held(ctx, res, &content, &patched, &held, say) + .await; + held.release(); + out + } + + async fn apply_held( + &self, + ctx: &Ctx, + res: &TuneResult, + content: &str, + patched: &str, + held: &Held<'_>, + say: &(dyn Fn(String) + Send + Sync), + ) -> Result { + let backup = self.home.join("tune").join("backup").join(format!( + "{}-{}.env", + res.preset_id, + unix_secs() + )); + if let Some(d) = backup.parent() { + let _ = mkdir_private(d); + } + write_private(&backup, content.as_bytes()) + .map_err(|e| format!("sauvegarde du preset : {e}"))?; + let backup = backup.to_string_lossy().into_owned(); + // Noté dans le verrou AVANT d'écrire : un Loki tué pendant + // l'application la défait au redémarrage. + held.set_backup(&backup, &res.preset_id, &res.preset_name); + let backup = &backup; + let revert = |why: String| async move { + say(format!("échec ({why}) : retour à l'ancienne version")); + if let Err(e) = crate::presets_api::save_applying( + ctx, + &res.preset_id, + &res.preset_name, + content, + "apply", + ) + .await + { + return Err(format!( + "{why} ; RETOUR IMPOSSIBLE ({e}) — l'ancienne version est dans {backup}" + )); + } + if let Err(e) = Self::svc_os(ctx, "restart").await { + return Err(format!( + "{why} ; ancienne version remise, redémarrage : {}", + crate::enginectl::plain_err(&e) + )); + } + Err(format!("{why} : ancienne version rétablie")) + }; + say(format!("écriture du preset (sauvegarde : {backup})")); + let applied = match crate::presets_api::save_applying( + ctx, + &res.preset_id, + &res.preset_name, + patched, + "apply", + ) + .await + { + Ok((_, applied)) => applied, + Err(e) => return revert(format!("écriture : {e}")).await, + }; + if !applied { + return revert("le preset n'a pas été appliqué à la configuration active".into()).await; + } + // Même configuration que l'essai retenu, à l'empreinte près. + let why = applied_mismatch(&ctx.store.config().await, res); + if !why.is_empty() { + return revert(why).await; + } + say("redémarrage du moteur".into()); + if let Err(e) = Self::svc_os(ctx, "restart").await { + return revert(format!("redémarrage : {}", crate::enginectl::plain_err(&e))).await; + } + if let Err(e) = self.post_apply_probe(ctx, res, say).await { + return revert(e).await; + } + Ok(format!( + "preset « {} » optimisé et vérifié", + res.preset_name + )) + } + + /// `tunePostApplyProbe` : le moteur redémarré répond, garde le contexte + /// et les slots mesurés, et tient un prompt à la profondeur de la mesure + /// suivi d'un decode. + async fn post_apply_probe( + &self, + ctx: &Ctx, + res: &TuneResult, + say: &(dyn Fn(String) + Send + Sync), + ) -> Result<(), String> { + say("attente du moteur".into()); + let cfg = ctx.store.config().await; + let (_tx, rx) = watch::channel(false); + let e = self.engine( + format!("http://localhost:{}", crate::engstatus::llm_port_raw(&cfg)), + Self::engine_key(ctx).await, + rx, + ); + let deadline = Instant::now() + self.load_timeout; + // Un moteur mort au chargement (OOM) se voit vite : plus de service du + // tout, trois fois de suite après les premières secondes. + let (mut gone, t0) = (0, Instant::now()); + while !crate::engstatus::health_check(&ctx.store.config().await).await { + if Instant::now() > deadline { + return Err(format!( + "le moteur ne répond pas après {}", + bench::go_dur(self.load_timeout) + )); + } + if t0.elapsed() > Duration::from_secs(10) && !Self::main_active(ctx).await { + gone += 1; + if gone >= 3 { + return Err("le moteur s'est arrêté au chargement".into()); + } + } else { + gone = 0; + } + tokio::time::sleep(self.probe_poll).await; + } + let (n_ctx, slots, _) = props(&e) + .await + .map_err(|e| format!("/props illisible : {e}"))?; + if n_ctx != res.n_ctx || slots != res.slots { + return Err(format!( + "contexte {n_ctx} × {slots} slot(s) au lieu de {} × {}", + res.n_ctx, res.slots + )); + } + if res.depth <= 0 { + return Ok(()); + } + say(format!( + "sonde : prompt de {} jetons puis decode", + res.depth + )); + let cfg = ctx.store.config().await; + let effort = self.effort(ctx); + let mut s = bench::setup_from(&cfg, &HashMap::new(), effort); + let code = Corpora::default_set().code; + let (sample, _) = bench::slice(&code, 0, bench::SAMPLE_BYTES as i64); + let ratio = e.bytes_per_token(sample).await; + let (text, _) = bench::slice(&code, 0, (res.depth as f64 * ratio) as i64); + let side = ctx + .store + .internal_post("/internal/v1/engine/sidejob", &json!({})) + .await + .ok() + .and_then(|(_, v)| v.get("id").and_then(Value::as_i64)); + let learn = |want: &str, got: &str| { + effort.remember(&cfg, want, got); + log_effort_fallback(want, got); + }; + let msgs = vec![( + "user".to_string(), + format!("{text}\n\n{}", bench::ASK_FIRST), + )]; + let r = s.ask(&e, &msgs, 64, false, &learn).await; + if let Some(id) = side { + let _ = ctx + .store + .internal_post( + "/internal/v1/engine/sidejob", + &json!({ "end": id, "served": e.served.load(Ordering::SeqCst) > 0 }), + ) + .await; + } + r.map(|_| ()).map_err(|e| format!("sonde : {e}")) + } + + /// `tuneEngineBin` puis `engineBuildCached` : le build du moteur de la + /// configuration, gardé tant que le binaire ne change pas. + async fn engine_build(&self, cfg: &Cfg) -> i64 { + let bin = cfg.get("BIN").map_or("", |s| s.trim()); + if bin.is_empty() { + return 0; + } + let bin = if crate::paths::go_is_abs(bin) { + bin.to_string() + } else { + crate::paths::go_join(&[&self.home.to_string_lossy(), bin]) + }; + let bin = crate::serve::run::EnginePaths { + home: self.home.clone(), + } + .prebuilt_resolve_bin(&bin); + let Ok(md) = std::fs::metadata(&bin) else { + return 0; + }; + let mtime = md + .modified() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |d| d.as_nanos()); + let key = format!("{bin}|{}|{mtime}", md.len()); + if let Some(b) = self + .builds + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(&key) + { + return *b; + } + let (b, _) = crate::engineimg::oci::build_of(&bin).await; + self.builds + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(key, b); + b + } + + /// `handleTuneLast`. + async fn last(&self, ctx: &Ctx, id: &str) -> Value { + let Some(res) = load_result(ctx, id).await else { + return json!({ "ok": false }); + }; + let mut out = Map::new(); + out.insert("ok".into(), json!(true)); + out.insert( + "result".into(), + serde_json::to_value(&res).unwrap_or(Value::Null), + ); + if let Ok(content) = crate::presets_api::read_preset(&self.home, id) { + out.insert( + "preset_changed".into(), + json!(crate::presets::preset_fingerprint(&content) != res.preset_fp), + ); + } + if res.engine_build > 0 { + let b = self.engine_build(&ctx.store.config().await).await; + if b > 0 && b != res.engine_build { + out.insert( + "engine_changed".into(), + json!(format!("b{} → b{b}", res.engine_build)), + ); + } + } + Value::Object(out) + } + + /// `handleTuneApply`. + async fn apply_route(self: &Arc, ctx: &Ctx, target: &str) -> (StatusCode, Value) { + let fail = |code: StatusCode, e: &str| (code, json!({ "ok": false, "error": e })); + let (res, busy) = { + let st = self.st(); + (st.result.clone(), st.running || st.applying || st.starting) + }; + if busy { + return fail(StatusCode::CONFLICT, ERR_TUNE_BUSY); + } + let Some(res) = res else { + return fail(StatusCode::BAD_REQUEST, "aucun résultat à appliquer"); + }; + if target == APPLY_COPY { + return match self.apply(ctx, &res, APPLY_COPY, &|_| {}).await { + Ok(msg) => (StatusCode::OK, json!({ "ok": true, "message": msg })), + Err(e) => fail(StatusCode::OK, &e), + }; + } + if target != APPLY_PRESET { + return fail(StatusCode::BAD_REQUEST, "cible inconnue"); + } + if let Err(e) = self.plan_apply(&res) { + return fail(StatusCode::OK, &e); + } + if crate::enginectl::generating(ctx).await { + let gc = Self::go_context(ctx).await; + let why = gc["busy"] + .as_str() + .filter(|s| !s.is_empty()) + .unwrap_or("génération en cours") + .to_string(); + return fail(StatusCode::CONFLICT, &why); + } + // Vérifié et posé d'un seul tenant : deux clics rapprochés passaient + // tous deux le test du début. + { + let mut st = self.st(); + if st.running || st.applying || st.starting { + return fail(StatusCode::CONFLICT, ERR_TUNE_BUSY); + } + st.applying = true; + st.apply_log.clear(); + st.apply_msg.clear(); + st.apply_err.clear(); + } + let (me, ctx2) = (self.clone(), ctx.clone()); + tokio::spawn(async move { + let me2 = me.clone(); + let say = move |s: String| me2.st().apply_log.push(s); + let out = me.apply(&ctx2, &res, APPLY_PRESET, &say).await; + let mut st = me.st(); + st.applying = false; + match out { + Ok(m) => st.apply_msg = m, + Err(e) => st.apply_err = e, + } + }); + (StatusCode::ACCEPTED, json!({ "ok": true })) + } +} + +// --------------------------------------------------------------------------- +// Les essais +// --------------------------------------------------------------------------- + +/// `tuneLoaded` : un essai chargé et prêt. +struct Loaded { + eng: Engine, + n_ctx: i64, + slots: i64, + build: String, + offloaded: i64, + layers: i64, + kv: String, +} + +/// L'arrêt d'un essai : le groupe entier, sa trace effacée du verrou, la VRAM +/// rendue. Sans processus : rien. +struct Stopper<'a> { + run: Option<(i64, Arc)>, + held: &'a Held<'a>, + settle: Duration, +} + +impl Stopper<'_> { + async fn stop(&mut self) { + let Some((pid, done)) = self.run.take() else { + return; + }; + let before = crate::enginectl::gpu_used_mb(&crate::sysinfo::gpu_stats().await); + let d = done.clone(); + let _ = tokio::task::spawn_blocking(move || { + lock::kill_tree(pid, &|| d.load(Ordering::SeqCst)); + }) + .await; + // Noté jusqu'à sa fin constatée : un essai qui survit à l'arrêt reste + // désigné dans le verrou, qu'un Loki redémarré pourra encore arrêter. + if done.load(Ordering::SeqCst) { + self.held.set_trial(None); + } else { + eprintln!("[loki tune] l'essai (PID {pid}) ne s'est pas arrêté dans les temps"); + } + if before > 0 { + gpu_settled(before).await; + } else { + tokio::time::sleep(self.settle).await; + } + } +} + +/// `tuneTrialer` : lance les essais d'une optimisation. +struct Trialer<'a> { + tune: &'a Tune, + dir: PathBuf, + seq: u32, + held: &'a Held<'a>, + key: String, +} + +impl<'a> Trialer<'a> { + /// `prepare` : la configuration d'un essai (copie + adresse privée) dans + /// un dossier neuf. + fn prepare(&mut self, cfg: &Cfg) -> Result<(PathBuf, u16), String> { + self.seq += 1; + let dir = self.dir.join(format!("e{:02}", self.seq)); + mkdir_private(&dir).map_err(|e| e.to_string())?; + let port = std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|l| l.local_addr()) + .map_err(|e| e.to_string())? + .port(); + let mut c = cfg.clone(); + c.insert("HOST".into(), "127.0.0.1".into()); + c.insert("PORT".into(), port.to_string()); + let text = tc::trial_config_text(&c)?; + write_private(&dir.join("config.env"), text.as_bytes()).map_err(|e| e.to_string())?; + Ok((dir, port)) + } + + fn command(&self, dir: &Path) -> tokio::process::Command { + let mut cmd = tokio::process::Command::new(&self.tune.go_bin); + cmd.arg("serve") + .current_dir(&self.tune.home) + .env(tc::TRIAL_ENV, dir) + .stdin(std::process::Stdio::null()); + for (k, v) in &self.tune.trial_env { + cmd.env(k, v); + } + cmd + } + + /// `dry` : « loki serve » à blanc — la ligne de commande, rien de chargé. + async fn dry(&self, cancel: &watch::Receiver, dir: &Path) -> Result { + let out_path = dir.join("dry.log"); + let out = std::fs::File::create(&out_path).map_err(|e| e.to_string())?; + let err = out.try_clone().map_err(|e| e.to_string())?; + let mut cmd = self.command(dir); + cmd.env(tc::DRY_RUN_ENV, "1") + .stdout(out) + .stderr(err) + .kill_on_drop(true); + let mut child = cmd.spawn().map_err(|e| e.to_string())?; + let mut cancel = cancel.clone(); + let status = tokio::select! { + s = tokio::time::timeout(Duration::from_secs(120), child.wait()) => match s { + Ok(Ok(s)) => s, + Ok(Err(e)) => return Err(e.to_string()), + Err(_) => { + let _ = child.kill().await; + return Err(last_line(&out_path, "signal: killed")); + } + }, + _ = async { + if cancel.wait_for(|c| *c).await.is_err() { + std::future::pending::<()>().await; + } + } => { + let _ = child.kill().await; + return Err("signal: killed".into()); + } + }; + if !status.success() { + let why = match status.code() { + Some(c) => format!("exit status {c}"), + None => "signal: killed".into(), + }; + return Err(last_line(&out_path, &why)); + } + let p = dir.join("launch.json"); + let b = std::fs::read(&p).map_err(|e| format!("open {}: {e}", p.display()))?; + serde_json::from_slice(&b).map_err(|e| e.to_string()) + } + + /// `load` : lance l'essai pour de vrai et attend qu'il réponde. Le + /// [`Stopper`] rendu l'arrête : à appeler quoi qu'il arrive. + async fn load( + &self, + cancel: &watch::Receiver, + dir: &Path, + port: u16, + launch: &Launch, + ) -> (Result, Stopper<'a>) { + let mut stopper = Stopper { + run: None, + held: self.held, + settle: self.tune.settle, + }; + let log_path = dir.join("engine.log"); + let (out, err) = + match std::fs::File::create(&log_path).and_then(|f| f.try_clone().map(|g| (f, g))) { + Ok(x) => x, + Err(e) => return (Err(e.to_string()), stopper), + }; + let mut cmd = self.command(dir); + cmd.stdout(out).stderr(err); + #[cfg(unix)] + cmd.process_group(0); + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(e) => return (Err(e.to_string()), stopper), + }; + let pid = i64::from(child.id().unwrap_or(0)); + let done = Arc::new(AtomicBool::new(false)); + let ended = Arc::new(Notify::new()); + { + let (done, ended) = (done.clone(), ended.clone()); + tokio::spawn(async move { + let _ = child.wait().await; + done.store(true, Ordering::SeqCst); + ended.notify_waiters(); + }); + } + let mut proc = Proc { + pid, + start: String::new(), + exes: vec![ + self.tune.go_bin.to_string_lossy().into_owned(), + launch.bin.clone(), + ], + }; + if let Some((start, _)) = lock::proc_identity(pid) { + proc.start = start; + } + self.held.set_trial(Some(proc)); + stopper.run = Some((pid, done.clone())); + let read_log = || { + std::fs::read(&log_path) + .map(|b| String::from_utf8_lossy(&b).into_owned()) + .unwrap_or_default() + }; + let (_tx, rx) = watch::channel(false); + let mut eng = self + .tune + .engine(format!("http://127.0.0.1:{port}"), self.key.clone(), rx); + eng.cancel = cancel.clone(); + let deadline = Instant::now() + self.tune.load_timeout; + let hc = reqwest::Client::builder() + .timeout(Duration::from_secs(3)) + .build() + .unwrap_or_default(); + let mut cancel = cancel.clone(); + loop { + let notified = ended.notified(); + if done.load(Ordering::SeqCst) { + return (Err(tc::load_failure(&read_log())), stopper); + } + tokio::select! { + _ = async { + if cancel.wait_for(|c| *c).await.is_err() { + std::future::pending::<()>().await; + } + } => return (Err("context canceled".into()), stopper), + _ = notified => return (Err(tc::load_failure(&read_log())), stopper), + _ = tokio::time::sleep(self.tune.poll) => {} + } + if let Ok(r) = hc.get(format!("{}/health", eng.base)).send().await + && r.status() == reqwest::StatusCode::OK + { + break; + } + if Instant::now() > deadline { + return ( + Err(format!( + "pas prêt après {}", + bench::go_dur(self.tune.load_timeout) + )), + stopper, + ); + } + } + let (n_ctx, slots, build) = match props(&eng).await { + Ok(p) => p, + Err(e) => return (Err(format!("/props illisible : {e}")), stopper), + }; + let (offloaded, layers, kv) = tc::log_facts(&read_log()); + ( + Ok(Loaded { + eng, + n_ctx, + slots, + build, + offloaded, + layers, + kv, + }), + stopper, + ) + } +} + +/// `tuneProps` : n_ctx, total_slots et build_info du moteur. +async fn props(e: &Engine) -> Result<(i64, i64, String), String> { + let mut req = e + .client + .get(format!("{}/props", e.base)) + .timeout(Duration::from_secs(10)); + if !e.key.is_empty() { + req = req.bearer_auth(&e.key); + } + let r = req.send().await.map_err(|e| e.to_string())?; + let status = r.status(); + let b = r.bytes().await.map_err(|e| e.to_string())?; + if status != reqwest::StatusCode::OK { + return Err(format!( + "GET /props : {} {}", + status.as_u16(), + status.canonical_reason().unwrap_or("") + )); + } + let v: Value = serde_json::from_slice(&b).map_err(|e| e.to_string())?; + let int = |v: &Value| v.as_i64().unwrap_or(0); + let mut n_ctx = int(&v["default_generation_settings"]["n_ctx"]); + if n_ctx <= 0 { + n_ctx = int(&v["n_ctx"]); + } + Ok(( + n_ctx, + int(&v["total_slots"]), + v["build_info"].as_str().unwrap_or("").to_string(), + )) +} + +/// La mesure d'un essai : le bench complet du preset de l'essai. +struct Measure<'a> { + tune: &'a Tune, + ctx: &'a Ctx, + tr: &'a Tracker, + env: &'a tc::Env, + cpu_placed: bool, + hybrid: bool, + weights: Weights, + bopts: bench::Opts, + corp: Corpora, +} + +impl Measure<'_> { + /// `n` passages ; rend ceux qui ont abouti et la première erreur. + async fn measure( + &self, + cfg: &Cfg, + l: &Loaded, + n: usize, + phase: &str, + ) -> (Vec, Option) { + let mut runs = Vec::new(); + let effort = self.tune.effort(self.ctx); + let learn = |want: &str, got: &str| { + effort.remember(cfg, want, got); + log_effort_fallback(want, got); + }; + for i in 0..n { + self.tr.set_phase(format!("{phase} : mesure {}/{n}", i + 1)); + // `tuneBenchSetup` : l'échantillonnage et le raisonnement réels du + // preset. SIDE_SLOT : les tours restent sur le second slot. + let mut s = bench::setup_from(cfg, &self.env.arg_env, effort); + s.cpu_placed = self.cpu_placed; + s.hybrid = self.hybrid; + s.side_slot = crate::serve::side::side_slot_on(cfg) && l.slots >= 2; + let r: Result = bench::run( + &l.eng, + self.bopts.clone(), + &mut s, + &self.corp, + &|_, _, _| {}, + &learn, + &|| None, + ) + .await; + let r = match r { + Ok(r) => r, + Err(e) => return (runs, Some(e)), + }; + match tc::run_from(Some(&r), &self.weights) { + Ok(run) => runs.push(run), + Err(e) => return (runs, Some(e)), + } + } + (runs, None) + } +} + +// --------------------------------------------------------------------------- +// Aides +// --------------------------------------------------------------------------- + +fn turn_secs(r: &[Run]) -> Vec { + r.iter().map(|x| x.turn_sec).collect() +} + +fn turn_mean(r: &[Run]) -> f64 { + tc::mean(&turn_secs(r)) +} + +fn prose_mean(r: &[Run]) -> f64 { + tc::mean(&r.iter().map(|x| x.prose_tg).collect::>()) +} + +/// `reBuildInfo` : `^b(\d+)`. +fn build_number(build: &str) -> i64 { + let d: String = build + .strip_prefix('b') + .unwrap_or("") + .chars() + .take_while(char::is_ascii_digit) + .collect(); + d.parse().unwrap_or(0) +} + +/// `Duration.Round(time.Minute)`. +fn round_minute(d: Duration) -> Duration { + let m = 60_000_000_000u128; + let n = d.as_nanos(); + let r = n % m; + let n = if r + r < m { n - r } else { n + m - r }; + Duration::from_nanos(n as u64) +} + +fn unix_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as i64) +} + +/// `tunePhysCores` : cœurs physiques (Linux ; ailleurs inconnu). +fn phys_cores(probe: &tc::Probe) -> i64 { + if probe.cpu_threads > 0 { + return probe.cpu_threads; + } + let root = Path::new("/"); + let Some(online) = crate::serve::threads::read_cpu_list(root, "sys/devices/system/cpu/online") + .filter(|o| !o.is_empty()) + else { + return 0; + }; + let atom = + crate::serve::threads::read_cpu_list(root, "sys/devices/cpu_atom/cpus").unwrap_or_default(); + crate::serve::threads::physical_cores(root, &online, &atom).unwrap_or(0) +} + +/// `gpuUsedSettled`. +async fn gpu_settled(before: i64) -> i64 { + crate::enginectl::gpu_settle( + before, + || async { + let g = crate::sysinfo::gpu_stats().await; + (crate::enginectl::gpu_used_mb(&g), !g.is_empty()) + }, + Duration::from_millis(500), + ) + .await +} + +/// La dernière ligne de la sortie, sans couleurs ; vide : `fallback`. +fn last_line(path: &Path, fallback: &str) -> String { + let out = std::fs::read(path) + .map(|b| String::from_utf8_lossy(&b).into_owned()) + .unwrap_or_default(); + let msg = crate::enginectl::plain_err(&out); + let msg = msg.trim(); + let msg = msg.rsplit('\n').next().unwrap_or(""); + if msg.is_empty() { + fallback.to_string() + } else { + msg.to_string() + } +} + +fn mkdir_private(d: &Path) -> std::io::Result<()> { + let mut b = std::fs::DirBuilder::new(); + b.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + b.mode(0o700); + } + b.create(d) +} + +fn write_private(p: &Path, b: &[u8]) -> std::io::Result<()> { + let mut o = std::fs::OpenOptions::new(); + o.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + o.mode(0o600); + } + use std::io::Write as _; + o.open(p)?.write_all(b) +} + +/// `tuneAppliedMismatch` : la configuration active après application doit +/// être EXACTEMENT la référence mesurée plus les clés réglées. "" = conforme. +pub fn applied_mismatch(now: &Cfg, res: &TuneResult) -> String { + let mut back = now.clone(); + for (k, v) in &res.set { + if now.get(k).map_or("", String::as_str) != v { + return format!("la configuration appliquée diffère de l'essai retenu sur {k}"); + } + match res.old.get(k).map_or("", String::as_str) { + "" => { + back.remove(k); + } + old => { + back.insert(k.clone(), old.to_string()); + } + } + } + if crate::presets::config_fingerprint(&back) != res.base_fp { + return "la configuration a changé ailleurs que sur les clés réglées depuis la mesure" + .into(); + } + String::new() +} + +/// `saveTuneResult` : le dernier résultat par preset. Les autres entrées +/// gardent leurs octets. +async fn save_result(ctx: &Ctx, res: &TuneResult) { + let mut m: BTreeMap> = ctx + .store + .get(BK_STATE, RESULTS_KEY) + .await + .ok() + .flatten() + .and_then(|b| serde_json::from_slice(&b).ok()) + .unwrap_or_default(); + let Ok(raw) = serde_json::to_string(res).and_then(RawValue::from_string) else { + return; + }; + m.insert(res.preset_id.clone(), raw); + let _ = ctx + .store + .put(BK_STATE, RESULTS_KEY, gojson::marshal(&m).into_bytes()) + .await; +} + +/// `loadTuneResult`. +async fn load_result(ctx: &Ctx, id: &str) -> Option { + let b = ctx.store.get(BK_STATE, RESULTS_KEY).await.ok().flatten()?; + let mut m: HashMap = serde_json::from_slice(&b).ok()?; + m.remove(id) +} + +// --------------------------------------------------------------------------- +// Les routes +// --------------------------------------------------------------------------- + +pub fn handles(path: &str) -> bool { + matches!( + path, + "/api/tune" + | "/api/tune/status" + | "/api/tune/cancel" + | "/api/tune/apply" + | "/api/tune/last" + ) +} + +/// `postOnly`. +fn post_only(method: &Method) -> Option> { + if method == Method::POST { + return None; + } + let mut r = send_json( + StatusCode::METHOD_NOT_ALLOWED, + &json!({ "ok": false, "error": format!("méthode {method} refusée : POST attendu") }), + ); + r.headers_mut().insert( + hyper::header::ALLOW, + hyper::header::HeaderValue::from_static("POST"), + ); + Some(r) +} + +/// `from_go` : remise par le Go (tunnel), déjà authentifiée par lui. +pub async fn serve( + ctx: &Ctx, + tune: &Arc, + req: Request, + from_go: bool, +) -> Response { + let (parts, body) = req.into_parts(); + if !from_go && let Err(refus) = ctx.guard(&parts.headers).await { + return *refus; + } + let read = || async { + Limited::new(body, 1 << 16) + .collect() + .await + .map(|b| b.to_bytes().to_vec()) + .unwrap_or_default() + }; + match parts.uri.path() { + "/api/tune" => { + if let Some(r) = post_only(&parts.method) { + return r; + } + // Les erreurs de décodage sont ignorées (champs facultatifs). + let d = gojson::decode_struct( + &read().await, + TUNE_REQ, + &[ + ("placement", Kind::Bool), + ("optin", Kind::Bool), + ("budget_min", Kind::Int), + ("stages", Kind::StrList), + ], + ); + let mut opts = Opts { + placement: d.bool("placement"), + opt_in: d.bool("optin"), + stages: d.str_list("stages").unwrap_or_default(), + ..Default::default() + }; + if d.int("budget_min") > 0 { + opts.budget = Duration::from_secs(d.int("budget_min") as u64 * 60); + } + let (code, out) = tune.start(ctx, opts).await; + send_json(code, &out) + } + "/api/tune/status" => send_json(StatusCode::OK, &tune.status()), + "/api/tune/cancel" => { + if let Some(r) = post_only(&parts.method) { + return r; + } + send_json(StatusCode::OK, &json!({ "ok": tune.cancel() })) + } + "/api/tune/apply" => { + if let Some(r) = post_only(&parts.method) { + return r; + } + let d = gojson::decode_struct(&read().await, APPLY_REQ, &[("target", Kind::Str)]); + let (code, out) = tune.apply_route(ctx, &d.str("target")).await; + send_json(code, &out) + } + "/api/tune/last" => { + let id = url::form_urlencoded::parse(parts.uri.query().unwrap_or("").as_bytes()) + .find(|(k, _)| k == "id") + .map(|(_, v)| v.into_owned()) + .unwrap_or_default(); + send_json(StatusCode::OK, &tune.last(ctx, &id).await) + } + _ => send_json( + StatusCode::NOT_FOUND, + &json!({ "ok": false, "error": "route inconnue" }), + ), + } +} diff --git a/rust/loki-rs/src/tune/lock.rs b/rust/loki-rs/src/tune/lock.rs new file mode 100644 index 0000000..6f50b87 --- /dev/null +++ b/rust/loki-rs/src/tune/lock.rs @@ -0,0 +1,615 @@ +//! Verrou de l'optimiseur — port de `backend_tune_lock.go`, au même format. +//! +//! Pendant une optimisation, la VRAM appartient au moteur d'ESSAI : relancer +//! le vrai moteur à côté chargerait deux modèles sur des cartes qui n'en +//! tiennent qu'un. Le verrou est un FICHIER de LOKI_HOME, créé de façon +//! exclusive, que tout le monde lit : le Go (`tuneGuard`, tous ses refus), un +//! « loki tune » en ligne de commande, et cette façade. +//! +//! Le fichier porte l'identité du processus qui le tient (PID, instant de +//! démarrage, binaire) et celle de l'essai en cours. Un verrou dont le +//! propriétaire est mort est périmé : l'essai orphelin est arrêté — seulement +//! si PID, instant de démarrage ET binaire concordent — puis le fichier est +//! retiré. L'identité est écrite comme le Go l'écrit (`procIdentity`) : la +//! reprise du Go (`tuneRecoverWatch`, toutes les 30 s) doit reconnaître la +//! façade vivante, sans quoi elle écarterait son verrou en pleine mesure. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +/// `tuneLockName`. +pub const LOCK_NAME: &str = "tune.lock"; + +/// `tuneLockFreshness` : un verrou illisible plus jeune que ça est en cours +/// d'écriture, pas abandonné. +pub const FRESHNESS: Duration = Duration::from_secs(10); + +/// `errTuneBusy`. +pub const ERR_TUNE_BUSY: &str = "optimisation en cours (loki tune) : le moteur est réservé aux essais — attends la fin ou annule-la"; + +/// `tuneProc` : un processus désigné sans ambiguïté. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Proc { + pub pid: i64, + /// `procIdentity` ; vide = illisible au moment de l'écrire. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub start: String, + /// Binaires admis (un « loki serve » devient llama-server par exec). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exes: Vec, +} + +/// `tuneOwner` : le contenu du verrou. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Owner { + pub owner: Proc, + /// « web », « cli » ou « apply ». + #[serde(default)] + pub via: String, + /// « essais », « application », « relance », « rendu ». + #[serde(default)] + pub phase: String, + #[serde(default)] + pub since: i64, + /// Distingue deux verrous pris par le même processus dans la même seconde. + #[serde(default, skip_serializing_if = "is_zero")] + pub nonce: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trial: Option, + /// Le vrai moteur tournait avant l'optimisation. + #[serde(default)] + pub main_was_active: bool, + /// Application en cours : la version d'avant du preset, sauvée avant + /// toute écriture. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub backup: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub preset_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub preset_name: String, +} + +fn is_zero(v: &i64) -> bool { + *v == 0 +} + +/// `tuneHeldKey`. +pub fn held_key(o: &Owner) -> String { + format!("{}|{}|{}", o.via, o.since, o.nonce) +} + +/// `sameTuneLock` : la même prise de verrou, quelle que soit sa phase. +pub fn same_lock(a: &Owner, b: &Owner) -> bool { + a.owner.pid == b.owner.pid && a.owner.start == b.owner.start && held_key(a) == held_key(b) +} + +/// `sameExe` : deux binaires comparés par leur nom de fichier (sans .exe, +/// sans casse). +pub fn same_exe(a: &str, b: &str) -> bool { + let norm = |s: &str| { + let s = s.trim(); + let s = s.strip_suffix(" (deleted)").unwrap_or(s); + let s = s.replace('\\', "/"); + let base = go_base(&s).to_lowercase(); + base.strip_suffix(".exe").unwrap_or(&base).to_string() + }; + !a.is_empty() && !b.is_empty() && norm(a) == norm(b) +} + +/// `filepath.Base` (séparateur `/`). +fn go_base(p: &str) -> String { + let t = p.trim_end_matches('/'); + if t.is_empty() { + return if p.is_empty() { ".".into() } else { "/".into() }; + } + t.rsplit('/').next().unwrap_or(t).to_string() +} + +/// `readTuneOwner` : illisible, ou sans PID : pas de verrou. +pub fn read_owner(path: &Path) -> Option { + let b = std::fs::read(path).ok()?; + let o: Owner = serde_json::from_slice(&b).ok()?; + (o.owner.pid > 0).then_some(o) +} + +// --------------------------------------------------------------------------- +// Identité d'un processus +// --------------------------------------------------------------------------- + +/// `procIdentity` (Linux) : instant de démarrage (champ 22 de +/// `/proc//stat`, préfixé de l'identifiant du démarrage de la machine) +/// et binaire. Un zombie a fini de tourner : il ne compte plus. +#[cfg(target_os = "linux")] +pub fn proc_identity(pid: i64) -> Option<(String, String)> { + proc_identity_in(Path::new("/"), pid) +} + +/// [`proc_identity`] sous la racine `root` (les tests). +#[cfg(target_os = "linux")] +pub fn proc_identity_in(root: &Path, pid: i64) -> Option<(String, String)> { + if pid <= 0 { + return None; + } + let s = std::fs::read_to_string(root.join(format!("proc/{pid}/stat"))).ok()?; + let st = crate::engine::proctree::parse_proc_stat(&s)?; + if st.state == "Z" { + return None; + } + let exe = std::fs::read_link(root.join(format!("proc/{pid}/exe"))) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + // Binaire remplacé par une mise à jour pendant que le processus tourne : + // le lien porte « (deleted) », c'est pourtant bien le même processus. + let exe = exe.strip_suffix(" (deleted)").unwrap_or(&exe).to_string(); + let mut start = st.start; + if let Ok(b) = std::fs::read_to_string(root.join("proc/sys/kernel/random/boot_id")) { + let id = b.trim(); + if !id.is_empty() { + start = format!("{id}/{start}"); + } + } + Some((start, exe)) +} + +/// `procIdentity` (autres Unix) : `ps`, langue et fuseau fixés — le +/// processus web et un terminal en français doivent lire la même chaîne. +#[cfg(all(unix, not(target_os = "linux")))] +pub fn proc_identity(pid: i64) -> Option<(String, String)> { + if pid <= 0 || !pid_alive(pid) { + return None; + } + let p = pid.to_string(); + let out = std::process::Command::new("ps") + .args(["-o", "lstart=", "-p", &p]) + .env("LC_ALL", "C") + .env("LANG", "C") + .env("TZ", "UTC") + .output() + .ok()?; + if !out.status.success() { + return None; + } + let start = String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .collect::>() + .join(" "); + if start.is_empty() { + return None; + } + let exe = std::process::Command::new("ps") + .args(["-o", "comm=", "-p", &p]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .unwrap_or_default(); + Some((start, exe)) +} + +/// Hors Unix, la façade ne tient pas l'optimiseur (`main.rs`). +#[cfg(not(unix))] +pub fn proc_identity(_pid: i64) -> Option<(String, String)> { + None +} + +/// `processAlive` : `kill(pid, 0)`. +#[cfg(unix)] +pub fn pid_alive(pid: i64) -> bool { + let Ok(p) = i32::try_from(pid) else { + return false; + }; + // SAFETY: kill(2) au signal 0 n'envoie rien ; un PID absent rend -1. + p > 0 && unsafe { libc::kill(p, 0) == 0 } +} + +#[cfg(not(unix))] +pub fn pid_alive(_pid: i64) -> bool { + false +} + +/// `tuneProcAlive` : `p` désigne-t-il encore le MÊME processus ? +pub fn proc_alive(p: &Proc) -> bool { + let Some((start, exe)) = proc_identity(p.pid) else { + return false; + }; + if !p.start.is_empty() && start != p.start { + return false; + } + if exe.is_empty() || p.exes.is_empty() { + return true; + } + p.exes.iter().any(|e| same_exe(e, &exe)) +} + +/// `selfProc` : ce processus, tel que le verrou le note. +pub fn self_proc() -> Proc { + let pid = i64::from(std::process::id()); + let mut p = Proc { + pid, + ..Default::default() + }; + if let Some((start, exe)) = proc_identity(pid) { + p.start = start; + if !exe.is_empty() { + p.exes = vec![exe]; + } + } + if p.exes.is_empty() + && let Ok(me) = std::env::current_exe() + { + p.exes = vec![me.to_string_lossy().into_owned()]; + } + p +} + +/// `tuneKillTree` (Unix) : SIGTERM au groupe de l'essai, puis SIGKILL s'il +/// traîne. `gone` dit si le processus a disparu (récolté par l'appelant, ou +/// sondé pour un orphelin d'un ancien Loki). +#[cfg(unix)] +pub fn kill_tree(pid: i64, gone: &dyn Fn() -> bool) { + let Ok(p) = i32::try_from(pid) else { + return; + }; + if p <= 0 { + return; + } + let wait = |d: Duration| { + let deadline = std::time::Instant::now() + d; + while std::time::Instant::now() < deadline { + if gone() { + return true; + } + std::thread::sleep(Duration::from_millis(100)); + } + false + }; + // SAFETY: kill(2) n'a pas d'effet mémoire. + unsafe { + if libc::kill(-p, libc::SIGTERM) != 0 { + libc::kill(p, libc::SIGTERM); + } + } + if wait(Duration::from_secs(10)) { + return; + } + // SAFETY: idem. + unsafe { + libc::kill(-p, libc::SIGKILL); + } + wait(Duration::from_secs(5)); +} + +#[cfg(not(unix))] +pub fn kill_tree(_pid: i64, _gone: &dyn Fn() -> bool) {} + +// --------------------------------------------------------------------------- +// Le verrou +// --------------------------------------------------------------------------- + +/// Qui est vivant : [`proc_alive`] hors tests. +pub type AliveFn = Box bool + Send + Sync>; + +/// Le fichier de verrou de CE processus : les prises en cours (`tuneHeld`) +/// et le verrou des opérations sur le fichier (`tuneFileMu`). +pub struct LockFile { + pub path: PathBuf, + held: Mutex>, + file_mu: Mutex<()>, + alive: AliveFn, + pid: i64, + me: Proc, +} + +/// Une prise de verrou à défaire après le ménage d'un verrou périmé : l'essai +/// orphelin à arrêter et l'application à défaire sont à l'appelant. +pub type Claimed = Owner; + +impl LockFile { + pub fn new(home: &Path) -> Self { + Self::with(home, Box::new(proc_alive), self_proc()) + } + + /// `alive` et `me` remplaçables par les tests. + pub fn with(home: &Path, alive: AliveFn, me: Proc) -> Self { + LockFile { + path: home.join(LOCK_NAME), + held: Mutex::new(HashSet::new()), + file_mu: Mutex::new(()), + alive, + pid: me.pid, + me, + } + } + + fn held(&self) -> std::sync::MutexGuard<'_, HashSet> { + self.held.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn file(&self) -> std::sync::MutexGuard<'_, ()> { + self.file_mu.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// `tuneOwnerAlive` : un verrou qui porte NOTRE PID sans être l'un des + /// nôtres vient d'un processus mort dont le PID nous est échu. + pub fn owner_alive(&self, o: &Owner) -> bool { + if o.owner.pid == self.pid && !self.held().contains(&held_key(o)) { + return false; + } + (self.alive)(&o.owner) + } + + pub fn proc_alive(&self, p: &Proc) -> bool { + (self.alive)(p) + } + + /// `tuneActive` : une optimisation tient-elle le moteur ? Un verrou + /// périmé ne compte pas. + pub fn active(&self) -> Option { + let o = read_owner(&self.path)?; + self.owner_alive(&o).then_some(o) + } + + /// `tuneLockAcquire`, premier temps : la prise, ou qui la tient. `Err(None)` + /// : un verrou périmé à écarter ([`Self::reap_claim`]) avant de réessayer. + fn try_acquire(&self, owner: &Owner) -> Result<(), Option> { + let _f = self.file(); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o644); + } + match opts.open(&self.path) { + Ok(mut f) => { + use std::io::Write as _; + let b = serde_json::to_vec(owner).unwrap_or_default(); + let w = f.write_all(&b).and_then(|_| f.sync_all()); + drop(f); + if let Err(e) = w { + let _ = std::fs::remove_file(&self.path); + return Err(Some(format!("verrou de l'optimiseur : {e}"))); + } + Ok(()) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Err(None), + Err(e) => Err(Some(e.to_string())), + } + } + + /// `tuneLockAcquire` : prend le verrou, ou dit qui le tient. `reap` écarte + /// un verrou périmé (essai orphelin et application interrompue compris). + pub async fn acquire( + &self, + via: &str, + main_active: bool, + reap: F, + ) -> Result, String> + where + F: Fn() -> Fut, + Fut: std::future::Future, + { + if let Some(dir) = self.path.parent() { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let owner = Owner { + owner: self.me.clone(), + via: via.into(), + phase: "essais".into(), + since: now.as_secs() as i64, + nonce: now.as_nanos() as i64, + main_was_active: main_active, + ..Default::default() + }; + // Inscrit comme nôtre AVANT que le fichier n'existe : la reprise tient + // pour mort un verrou qui porte notre PID sans être inscrit. + let key = held_key(&owner); + self.held().insert(key.clone()); + for _ in 0..3 { + match self.try_acquire(&owner) { + Ok(()) => { + return Ok(Held { + file: self, + owner: Mutex::new(owner), + held: Mutex::new(true), + }); + } + Err(Some(e)) => { + self.held().remove(&key); + return Err(e); + } + Err(None) => {} + } + if let Some(o) = self.active() { + self.held().remove(&key); + let c = crate::tz::Location::local().civil(o.since); + return Err(format!( + "une optimisation tourne déjà ({}, PID {}, depuis {:02}:{:02})", + o.via, o.owner.pid, c.hour, c.minute + )); + } + reap().await; + } + self.held().remove(&key); + Err("verrou de l'optimiseur indisponible : réessaie".into()) + } + + /// `tuneReapClaim` : la partie du ménage qui touche au fichier — lecture, + /// déplacement, relecture, remise en place. Rend le verrou périmé + /// effectivement retiré (`None` : rien à écarter). + pub fn reap_claim(&self) -> Option { + let _f = self.file(); + let Some(o) = read_owner(&self.path) else { + // Illisible (écriture interrompue) : périmé lui aussi — mais pas + // tout juste créé (un verrou neuf est vide quelques instants). + if let Ok(md) = std::fs::metadata(&self.path) + && md + .modified() + .ok() + .and_then(|m| m.elapsed().ok()) + .is_some_and(|age| age > FRESHNESS) + { + let _ = std::fs::remove_file(&self.path); + } + return None; + }; + if self.owner_alive(&o) { + return None; + } + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let stale = PathBuf::from(format!( + "{}.stale-{}-{nanos}", + self.path.display(), + self.pid + )); + std::fs::rename(&self.path, &stale).ok()?; + let back = |stale: &Path| { + // Un autre verrou, vivant, déplacé par erreur : remis en place (un + // lien : jamais par-dessus un verrou posé entre-temps). + if let Err(e) = std::fs::hard_link(stale, &self.path) { + eprintln!( + "[loki tune] verrou vivant déplacé par erreur, remise en place impossible : {e}" + ); + } + }; + let got = read_owner(&stale); + let claimed = match got { + Some(g) if same_lock(&g, &o) && !self.owner_alive(&g) => Some(o), + _ => { + back(&stale); + None + } + }; + let _ = std::fs::remove_file(&stale); + claimed + } +} + +/// Le verrou tenu par CE processus (`tuneLock`). +pub struct Held<'a> { + file: &'a LockFile, + owner: Mutex, + held: Mutex, +} + +impl Held<'_> { + fn owner(&self) -> std::sync::MutexGuard<'_, Owner> { + self.owner.lock().unwrap_or_else(|e| e.into_inner()) + } + + pub fn snapshot(&self) -> Owner { + self.owner().clone() + } + + /// `writeLocked` : réécrit d'un bloc (fichier voisin puis renommage) — un + /// lecteur ne voit jamais un JSON à moitié écrit. + fn write_locked(&self, o: &Owner) -> std::io::Result<()> { + let b = serde_json::to_vec(o).map_err(std::io::Error::other)?; + let tmp = PathBuf::from(format!( + "{}.tmp-{}", + self.file.path.display(), + self.file.pid + )); + std::fs::write(&tmp, b)?; + let mut i = 0; + loop { + match std::fs::rename(&tmp, &self.file.path) { + Ok(()) => return Ok(()), + Err(e) if i == 4 => { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + Err(_) => { + i += 1; + std::thread::sleep(Duration::from_millis(50)); + } + } + } + } + + fn update(&self, f: impl FnOnce(&mut Owner)) { + let mut o = self.owner(); + f(&mut o); + let _f = self.file.file(); + let _ = self.write_locked(&o); + } + + /// `setTrial` : l'essai en cours (`None` : aucun), noté AVANT qu'il ne + /// charge quoi que ce soit — c'est ce qui permet de l'arrêter après un + /// redémarrage. + pub fn set_trial(&self, p: Option) { + self.update(|o| o.trial = p); + } + + pub fn set_phase(&self, phase: &str) { + self.update(|o| o.phase = phase.into()); + } + + /// `setBackup` : la sauvegarde du preset avant son écriture. + pub fn set_backup(&self, backup: &str, id: &str, name: &str) { + self.update(|o| { + o.backup = backup.into(); + o.preset_id = id.into(); + o.preset_name = name.into(); + }); + } + + /// `release` : rend le verrou — seulement s'il est encore le nôtre. Réécrit + /// d'abord NEUTRE (rien à défaire, rien à relancer, aucun essai), puis + /// retiré ; l'inscription ne part qu'après. + pub fn release(&self) { + let mut o = self.owner(); + let _f = self.file.file(); + let mut held = self.held.lock().unwrap_or_else(|e| e.into_inner()); + if *held { + let mut ours = false; + for _ in 0..5 { + if let Some(cur) = read_owner(&self.file.path) { + ours = same_lock(&cur, &o); + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + if ours { + o.phase = "rendu".into(); + o.trial = None; + o.main_was_active = false; + o.backup.clear(); + o.preset_id.clear(); + o.preset_name.clear(); + let _ = self.write_locked(&o); + for _ in 0..5 { + match std::fs::remove_file(&self.file.path) { + Ok(()) => break, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => break, + Err(_) => std::thread::sleep(Duration::from_millis(20)), + } + } + } + *held = false; + self.file.held().remove(&held_key(&o)); + return; + } + if read_owner(&self.file.path).is_some_and(|cur| same_lock(&cur, &o)) { + let _ = std::fs::remove_file(&self.file.path); + } + } +} + +impl Drop for Held<'_> { + fn drop(&mut self) { + if *self.held.lock().unwrap_or_else(|e| e.into_inner()) { + self.release(); + } + } +} diff --git a/rust/loki-rs/src/tune/mod.rs b/rust/loki-rs/src/tune/mod.rs new file mode 100644 index 0000000..2ac7f47 --- /dev/null +++ b/rust/loki-rs/src/tune/mod.rs @@ -0,0 +1,15 @@ +//! Optimiseur sans perte tenu par la façade (lot 28 de la migration, +//! `LOKI_TUNE=rust` ; expérimental, le Go reste le défaut) — port de +//! `backend_tune*.go`. +//! +//! - [`core`] : la partie pure (essais, garde-fous, score, réécriture du +//! preset) ; +//! - [`lock`] : le verrou de LOKI_HOME, au format du Go ; +//! - [`job`] : les routes, le job, les essais et l'application. + +pub mod core; +pub mod job; +pub mod lock; + +#[cfg(test)] +mod tests; diff --git a/rust/loki-rs/src/tune/tests.rs b/rust/loki-rs/src/tune/tests.rs new file mode 100644 index 0000000..e63c4b1 --- /dev/null +++ b/rust/loki-rs/src/tune/tests.rs @@ -0,0 +1,294 @@ +//! Tests de l'optimiseur : ceux du verrou (`backend_tune_test.go`), portés, +//! et les aides du job. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; + +use super::lock::*; + +fn me() -> Proc { + Proc { + pid: i64::from(std::process::id()), + start: "moi".into(), + exes: vec![], + } +} + +/// `TestTuneLock` : le verrou se prend, bloque, note l'essai, se rend ; un +/// propriétaire mort le rend périmé et le ménage le retire. +#[tokio::test] +async fn verrou() { + let tmp = tempfile::tempdir().unwrap(); + let alive = Arc::new(AtomicBool::new(true)); + let a = alive.clone(); + let pid = me().pid; + let lf = LockFile::with( + tmp.path(), + Box::new(move |p: &Proc| a.load(Ordering::SeqCst) && p.pid == pid), + me(), + ); + let no_reap = || async { false }; + assert!(lf.active().is_none(), "garde levée sans verrou"); + let l = lf.acquire("cli", true, no_reap).await.unwrap(); + assert!(lf.active().is_some(), "garde muette pendant l'optimisation"); + let e = lf.acquire("web", false, no_reap).await.err().unwrap(); + assert!( + e.starts_with(&format!( + "une optimisation tourne déjà (cli, PID {pid}, depuis " + )), + "{e}" + ); + l.set_trial(Some(Proc { + pid: 999999, + start: "x".into(), + exes: vec![], + })); + let o = read_owner(&lf.path).unwrap(); + assert_eq!(o.trial.as_ref().map(|t| t.pid), Some(999999)); + assert!(o.main_was_active && o.phase == "essais" && o.via == "cli"); + l.set_phase("relance"); + l.set_backup("/b", "Dense", "Dense"); + let o = read_owner(&lf.path).unwrap(); + assert_eq!( + (o.phase.as_str(), o.backup.as_str(), o.preset_id.as_str()), + ("relance", "/b", "Dense") + ); + l.release(); + assert!(!lf.path.exists(), "verrou non rendu"); + + // Propriétaire mort : périmé, il ne bloque rien et cède la place. + let l = lf.acquire("cli", true, no_reap).await.unwrap(); + l.set_trial(Some(Proc { + pid: 999999, + start: "x".into(), + exes: vec![], + })); + alive.store(false, Ordering::SeqCst); + assert!(lf.active().is_none(), "verrou périmé encore bloquant"); + let claimed = lf.reap_claim().expect("verrou périmé non écarté"); + assert!(claimed.main_was_active, "MainWasActive perdu au nettoyage"); + assert!(!lf.path.exists(), "verrou périmé non retiré"); + drop(l); + alive.store(true, Ordering::SeqCst); + let l2 = lf.acquire("web", false, no_reap).await.unwrap(); + // Un verrou rendu par sa poignée n'emporte pas celui d'un autre. + let other = l2.snapshot(); + l2.release(); + std::fs::write(&lf.path, serde_json::to_vec(&other).unwrap()).unwrap(); + assert!(lf.path.exists()); +} + +/// Un verrou périmé dans la place, et `acquire` qui le fait écarter. +#[tokio::test] +async fn verrou_perime_ecarte_a_la_prise() { + let tmp = tempfile::tempdir().unwrap(); + let lf = LockFile::with(tmp.path(), Box::new(|_: &Proc| false), me()); + let dead = Owner { + owner: Proc { + pid: 999999, + start: "x".into(), + exes: vec![], + }, + via: "cli".into(), + phase: "essais".into(), + main_was_active: true, + ..Default::default() + }; + std::fs::write(&lf.path, serde_json::to_vec(&dead).unwrap()).unwrap(); + let reaped = std::sync::Mutex::new(0); + let l = lf + .acquire("web", false, || async { + *reaped.lock().unwrap() += 1; + lf.reap_claim().is_some() + }) + .await + .unwrap(); + assert_eq!(*reaped.lock().unwrap(), 1); + assert_eq!(read_owner(&lf.path).unwrap().via, "web"); + drop(l); + assert!(!lf.path.exists(), "verrou lâché sans release : rendu"); +} + +/// `TestTuneLockStaleSelfAndFresh` : un verrou à notre PID qui n'est pas +/// l'un des nôtres est périmé ; illisible et neuf il reste, vieux il part. +#[tokio::test] +async fn verrou_perime_de_soi_et_frais() { + let tmp = tempfile::tempdir().unwrap(); + let lf = LockFile::with(tmp.path(), Box::new(|_: &Proc| true), me()); + let stale = Owner { + owner: me(), + via: "web".into(), + phase: "essais".into(), + ..Default::default() + }; + std::fs::write(&lf.path, serde_json::to_vec(&stale).unwrap()).unwrap(); + assert!( + lf.active().is_none(), + "verrou d'un ancien processus au même PID encore bloquant" + ); + let l = lf + .acquire("cli", false, || async { lf.reap_claim().is_some() }) + .await + .unwrap(); + assert!(lf.active().is_some(), "notre propre verrou ne bloque plus"); + let mine = l.snapshot(); + l.release(); + // Rendu : plus inscrit comme tenu, le même contenu est périmé. + std::fs::write(&lf.path, serde_json::to_vec(&mine).unwrap()).unwrap(); + assert!( + lf.active().is_none(), + "verrou rendu encore inscrit comme tenu" + ); + std::fs::remove_file(&lf.path).unwrap(); + + // Illisible et neuf : laissé en place. + std::fs::write(&lf.path, b"").unwrap(); + assert!(lf.reap_claim().is_none()); + assert!(lf.path.exists(), "verrou en cours d'écriture retiré"); + // Illisible et vieux : écarté. + let f = std::fs::File::options().write(true).open(&lf.path).unwrap(); + f.set_modified(SystemTime::now() - 2 * FRESHNESS).unwrap(); + drop(f); + assert!(lf.reap_claim().is_none()); + assert!(!lf.path.exists(), "verrou illisible abandonné non retiré"); +} + +/// Le ménage relit ce qu'il a déplacé : un verrou vivant pris entre la +/// lecture et le déplacement est remis en place. +#[tokio::test] +async fn menage_jamais_sur_un_verrou_vivant() { + let tmp = tempfile::tempdir().unwrap(); + let alive = Arc::new(AtomicBool::new(false)); + let a = alive.clone(); + let lf = LockFile::with( + tmp.path(), + Box::new(move |_: &Proc| a.load(Ordering::SeqCst)), + me(), + ); + let o = Owner { + owner: Proc { + pid: 999999, + start: "x".into(), + exes: vec![], + }, + via: "cli".into(), + ..Default::default() + }; + std::fs::write(&lf.path, serde_json::to_vec(&o).unwrap()).unwrap(); + alive.store(true, Ordering::SeqCst); + assert!(lf.reap_claim().is_none(), "verrou vivant écarté"); + assert_eq!(read_owner(&lf.path).unwrap(), o); + // Prises et rendus répétés pendant que le ménage tourne : jamais écarté. + let stop = Arc::new(AtomicBool::new(false)); + let lf = Arc::new(LockFile::with( + tmp.path(), + Box::new(|p: &Proc| p.pid == i64::from(std::process::id())), + me(), + )); + std::fs::remove_file(&lf.path).unwrap(); + let (s2, lf2) = (stop.clone(), lf.clone()); + let reaper = std::thread::spawn(move || { + let mut n = 0; + while !s2.load(Ordering::SeqCst) { + if lf2.reap_claim().is_some() { + n += 1; + } + } + n + }); + for i in 0..60 { + let l = lf + .acquire("apply", true, || async { false }) + .await + .unwrap_or_else(|e| panic!("tour {i} : {e}")); + l.set_phase("application"); + l.release(); + } + stop.store(true, Ordering::SeqCst); + assert_eq!( + reaper.join().unwrap(), + 0, + "verrou vivant de ce processus écarté" + ); +} + +/// `TestTuneSameExeDeleted`. +#[test] +fn binaire_remplace() { + assert!(same_exe( + "/usr/local/bin/loki", + "/usr/local/bin/loki (deleted)" + )); + assert!(!same_exe("/usr/local/bin/loki", "/usr/bin/llama-server")); + assert!(!same_exe("", "")); +} + +/// L'identité d'un processus comme le Go l'écrit : instant de démarrage +/// préfixé de l'identifiant du démarrage, binaire sans « (deleted) ». +#[cfg(target_os = "linux")] +#[test] +fn identite_d_un_processus() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let p = root.join("proc/77"); + std::fs::create_dir_all(&p).unwrap(); + std::fs::create_dir_all(root.join("proc/sys/kernel/random")).unwrap(); + let stat = |state: &str| { + format!("77 (llama (x) server) {state} 1 77 77 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 4242 0 0") + }; + std::fs::write(p.join("stat"), stat("S")).unwrap(); + std::os::unix::fs::symlink("/opt/llama-server (deleted)", p.join("exe")).unwrap(); + assert_eq!( + proc_identity_in(root, 77), + Some(("4242".into(), "/opt/llama-server".into())) + ); + std::fs::write(root.join("proc/sys/kernel/random/boot_id"), "b0c1\n").unwrap(); + assert_eq!(proc_identity_in(root, 77).unwrap().0, "b0c1/4242"); + std::fs::write(p.join("stat"), stat("Z")).unwrap(); + assert_eq!(proc_identity_in(root, 77), None, "zombie"); + assert_eq!(proc_identity_in(root, 0), None); + assert_eq!(proc_identity_in(root, 78), None); + // Ce processus : vivant, reconnu, et pas un autre au même PID. + let me = self_proc(); + assert!(proc_alive(&me)); + assert!(!proc_alive(&Proc { + start: format!("{}x", me.start), + ..me.clone() + })); + assert!(!proc_alive(&Proc { + exes: vec!["/usr/bin/autre".into()], + ..me.clone() + })); + assert!(proc_alive(&Proc { + exes: vec![], + start: String::new(), + ..me + })); +} + +/// `TestTuneReapOrphan` (la partie processus) : le groupe de l'essai est +/// arrêté, SIGTERM d'abord. +#[cfg(unix)] +#[test] +fn arret_du_groupe() { + use std::os::unix::process::CommandExt; + let mut child = std::process::Command::new("sleep") + .arg("60") + .process_group(0) + .spawn() + .unwrap(); + let pid = i64::from(child.id()); + assert!(pid_alive(pid)); + let t0 = std::time::Instant::now(); + let reaped = Arc::new(AtomicBool::new(false)); + let r = reaped.clone(); + let waiter = std::thread::spawn(move || { + let _ = child.wait(); + r.store(true, Ordering::SeqCst); + }); + kill_tree(pid, &|| reaped.load(Ordering::SeqCst)); + waiter.join().unwrap(); + assert!(t0.elapsed() < Duration::from_secs(5), "SIGTERM ignoré ?"); + assert!(!pid_alive(pid)); +} diff --git a/rust/loki-rs/tests/common/fakego.rs b/rust/loki-rs/tests/common/fakego.rs index f683fc3..1a64491 100644 --- a/rust/loki-rs/tests/common/fakego.rs +++ b/rust/loki-rs/tests/common/fakego.rs @@ -117,6 +117,14 @@ pub struct FakeGo { pub bench_perfs: Mutex>, /// Délai avant de répondre à une prise de bail. pub lease_delay: Mutex>, + /// /internal/v1/tune/context : refus du processus web et télémétrie + /// (« main_active » : `service_active`). + pub tune_ctx_busy: Mutex, + pub tune_perf: Mutex, + /// /internal/v1/service/action reçus avec « os » ; une erreur à rendre + /// pour une action donnée. + pub service_os: Mutex>, + pub service_os_fail: Mutex>, } impl FakeGo { @@ -470,6 +478,19 @@ async fn handle(st: Arc, req: Request) -> Response let body = req.into_body().collect().await.unwrap().to_bytes(); let v: serde_json::Value = serde_json::from_slice(&body).unwrap_or_default(); let if_active = v["if_active"].as_bool().unwrap_or(false); + if v["os"].as_bool() == Some(true) { + let action = v["action"].as_str().unwrap_or("").to_string(); + st.service_os.lock().unwrap().push(action.clone()); + let fail = st.service_os_fail.lock().unwrap().clone(); + if let Some((a, e)) = fail.filter(|(a, _)| *a == action) { + let _ = a; + return resp( + 200, + serde_json::json!({"ran": true, "error": e}).to_string(), + ); + } + return resp(200, r#"{"ran":true,"error":""}"#); + } st.service_actions .lock() .unwrap() @@ -479,6 +500,14 @@ async fn handle(st: Arc, req: Request) -> Response } resp(200, r#"{"ran":true,"error":""}"#) } + "/internal/v1/tune/context" => { + let v = serde_json::json!({ + "busy": *st.tune_ctx_busy.lock().unwrap(), + "main_active": *st.service_active.lock().unwrap(), + "perf": *st.tune_perf.lock().unwrap(), + }); + resp(200, v.to_string()) + } "/internal/v1/dictate/stop" => { *st.dictate_stops.lock().unwrap() += 1; resp(204, "") @@ -756,6 +785,7 @@ pub struct Opts { pub strata: Option>, pub tasks: Option>, pub bench: Option>, + pub tune: Option>, /// Base de départ, posée avant que la façade démarre : (bucket, clé, /// valeur). pub seed: Vec<(String, String, String)>, @@ -868,6 +898,7 @@ pub async fn start_with( strata: opts.strata, tasks: opts.tasks, bench: opts.bench, + tune: opts.tune, }; tokio::spawn(loki_rs::proxy::serve(fl, cfg, async move { let _ = rx.await; diff --git a/rust/loki-rs/tests/lot2.rs b/rust/loki-rs/tests/lot2.rs index 83863b2..2f62f7e 100644 --- a/rust/loki-rs/tests/lot2.rs +++ b/rust/loki-rs/tests/lot2.rs @@ -111,6 +111,7 @@ async fn banc(cle: Option<&str>) -> Banc { strata: None, tasks: None, bench: None, + tune: None, }; tokio::spawn(loki_rs::proxy::serve(l, cfg, async move { let _ = rx.await; diff --git a/rust/loki-rs/tests/parity_tune/core.json b/rust/loki-rs/tests/parity_tune/core.json new file mode 100644 index 0000000..d3edbff --- /dev/null +++ b/rust/loki-rs/tests/parity_tune/core.json @@ -0,0 +1,29822 @@ +{ + "applied": { + "base_fp": "f0a2be841ae1baac66a6bd72e4dd3077006e78bb", + "cases": [ + { + "now": { + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "why": "" + }, + { + "now": { + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "why": "la configuration appliquée diffère de l'essai retenu sur UBATCH" + }, + { + "now": { + "MODEL": "autre.gguf", + "UBATCH": "2048" + }, + "why": "la configuration a changé ailleurs que sur les clés réglées depuis la mesure" + }, + { + "now": { + "EXTRA_ARGS": "-t 4", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "why": "la configuration appliquée diffère de l'essai retenu sur EXTRA_ARGS" + }, + { + "now": { + "MODEL": "m.gguf", + "PORT": "9", + "UBATCH": "2048" + }, + "why": "la configuration a changé ailleurs que sur les clés réglées depuis la mesure" + } + ], + "old": { + "EXTRA_ARGS": "-t 8", + "UBATCH": "512" + }, + "set": { + "EXTRA_ARGS": "", + "UBATCH": "2048" + } + }, + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "--fit resterait inactif (--tensor-split)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 1 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "2", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=2", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "4", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "SPEC_N_MAX=4", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cfg_name": "riche", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 3 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=3", + "stage": "threads" + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 2 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "label": "THREADS=2", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "1" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "32768", + "FIT_TARGET": "1536", + "MODEL": "m.gguf" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "MODEL": "m.gguf" + }, + "cfg_name": "simple", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "1" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "cfg_name": "contexte non chiffré", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "3" + }, + "label": "THREADS=3", + "stage": "threads" + }, + { + "cfg": { + "EXTRA_ARGS": "-ot exps=CPU", + "MODEL": "m.gguf", + "THREADS": "2" + }, + "label": "THREADS=2", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "1" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (--tensor-split)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPEC": "auto", + "SPLIT_MODE": "tensor" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "SPLIT_MODE=tensor" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPLIT_MODE": "tensor" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "cfg_name": "split tensor", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "32768", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "32768", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ts 1,1", + "MODEL": "m.gguf", + "SPLIT_MODE": "tensor" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "1", + "UBATCH": "1024" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "auto", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "EXTRA_ARGS": "--backend-sampling", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "32", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=32", + "stage": "délestage" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "UBATCH": "1024" + }, + "label": "threads omis (cœurs physiques)", + "stage": "threads" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "4", + "UBATCH": "1024" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "cfg_name": "lots posés", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "CUDA_LAUNCH_QUEUES": "4X", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "THREADS": "7", + "UBATCH": "1024" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "1" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (couches --n-cpu-moe sur CPU)" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "-bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "cfg_name": "spéculation", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "draft", + "SPEC_N_MAX": "4" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "3" + }, + "label": "THREADS=3", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "On", + "EXTRA_ARGS": "--n-cpu-moe 10 -bs", + "MODEL": "m.gguf", + "SPEC": "draft", + "SPEC_N_MAX": "4", + "THREADS": "2" + }, + "label": "THREADS=2", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "1" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [], + "why": "rien à essayer" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "off", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "8192", + "CUDA_LAUNCH_QUEUES": "4x", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1024,2048", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1024,2048", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "2048,1024", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=2048,1024", + "stage": "marges" + }, + { + "cfg": { + "CTX": "8192", + "FIT_TARGET": "1536", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "FIT_TARGET=1536", + "stage": "marges" + } + ], + "why": "" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "8192", + "EXTRA_ARGS": "--backend-sampling", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "rien n'est placé à la main" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512", + "SPEC": "ngram" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "7" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "THREADS": "4" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "cfg_name": "ngram", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [], + "why": "tous les poids sur GPU" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "8192", + "MODEL": "m.gguf", + "SPEC": "ngram", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "8192", + "CUDA_GRAPH_OPT": "on", + "MODEL": "m.gguf", + "SPEC": "ngram" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [], + "why": "tous les poids sur GPU" + } + }, + "weights_on_cpu": false + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "riche", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans placement", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans --fit-target", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "ce moteur ne connaît pas --fit-target" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": false, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans nvidia-smi", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "VRAM libre non mesurable (nvidia-smi)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": false, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "sans opt-in", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "une carte", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 3, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "trois cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 0, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "cœurs inconnus", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [], + "why": "nombre de cœurs physiques inconnu" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 2, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "deux cœurs", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 1", + "MODEL": "m.gguf" + }, + "label": "THREADS=1", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": { + "LLAMA_ARG_BACKEND_SAMPLING": "1" + }, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 85899345920, + "moe": true, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": true, + "smi": true, + "user_env": { + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "64" + }, + "vram_mib": 28672 + }, + "env_name": "environnement posé", + "stages": { + "délestage": { + "cands": [], + "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" + }, + "files CUDA": { + "cands": [], + "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=4096", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "8192", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "4096" + }, + "label": "UBATCH=4096 BATCH=8192", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 2, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 10737418240, + "moe": false, + "opt_in": true, + "phys_cores": 8, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 28672 + }, + "env_name": "dense deux cartes", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "off", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=off", + "stage": "files CUDA" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_LAUNCH_QUEUES": "4x", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_LAUNCH_QUEUES=4x", + "stage": "files CUDA" + } + ], + "why": "" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "--fit inactif (surcharge de tenseurs)" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "SPEC": "auto" + }, + "label": "SPEC=auto", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", + "MODEL": "m.gguf" + }, + "label": "--backend-sampling", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-t 0", + "MODEL": "m.gguf" + }, + "label": "placement auto (--fit) au lieu des experts placés à la main", + "stage": "placement" + } + ], + "why": "" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 1, + "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", + "model_bytes": 21474836480, + "moe": false, + "opt_in": false, + "phys_cores": 8, + "placement": false, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 8192 + }, + "env_name": "modèle plus gros que la VRAM", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [], + "why": "non demandé (« inclure les options opt-in »)" + }, + "placement": { + "cands": [], + "why": "non demandé (« inclure le placement »)" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 7", + "MODEL": "m.gguf" + }, + "label": "THREADS=7", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 4", + "MODEL": "m.gguf" + }, + "label": "THREADS=4", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "cfg_name": "cpu-moe", + "env": { + "arg_env": null, + "gpus": 1, + "help": "", + "model_bytes": 0, + "moe": false, + "opt_in": true, + "phys_cores": 4, + "placement": true, + "queues_env": false, + "smi": true, + "user_env": null, + "vram_mib": 0 + }, + "env_name": "moteur ancien", + "stages": { + "délestage": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "128" + }, + "label": "OP_OFFLOAD_MIN_BATCH=128", + "stage": "délestage" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "OP_OFFLOAD_MIN_BATCH": "512" + }, + "label": "OP_OFFLOAD_MIN_BATCH=512", + "stage": "délestage" + } + ], + "why": "" + }, + "files CUDA": { + "cands": [], + "why": "une seule carte" + }, + "lots": { + "cands": [ + { + "cfg": { + "BATCH": "1024", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "512" + }, + "label": "UBATCH=512 BATCH=1024", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "1024" + }, + "label": "UBATCH=1024 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "2048", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=2048", + "stage": "lots" + }, + { + "cfg": { + "BATCH": "4096", + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf", + "UBATCH": "2048" + }, + "label": "UBATCH=2048 BATCH=4096", + "stage": "lots" + } + ], + "why": "" + }, + "marges": { + "cands": [], + "why": "une seule carte" + }, + "opt-in": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-cmoe -t 0", + "MODEL": "m.gguf" + }, + "label": "CUDA_GRAPH_OPT=on", + "stage": "opt-in" + } + ], + "why": "" + }, + "placement": { + "cands": [], + "why": "ce moteur ne connaît pas --fit" + }, + "threads": { + "cands": [ + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 3", + "MODEL": "m.gguf" + }, + "label": "THREADS=3", + "stage": "threads" + }, + { + "cfg": { + "CTX": "16384", + "EXTRA_ARGS": "-cmoe -t 2", + "MODEL": "m.gguf" + }, + "label": "THREADS=2", + "stage": "threads" + } + ], + "why": "" + } + }, + "weights_on_cpu": true + } + ], + "cfg_check": [ + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "", + "check_optin": "", + "diff": null, + "name": "identique" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20", + "UBATCH": "2048" + }, + "check": "", + "check_optin": "", + "diff": [ + "BATCH : (absent) → 4096", + "UBATCH : (absent) → 2048" + ], + "name": "lots" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "32768", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "clé CTX intouchable", + "check_optin": "clé CTX intouchable", + "diff": [ + "CTX : 65536 → 32768" + ], + "name": "contexte" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "auto", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "clé SPEC intouchable", + "check_optin": "", + "diff": [ + "SPEC : mtp → auto" + ], + "name": "spec" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "CUDA_GRAPH_OPT": "on", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "clé CUDA_GRAPH_OPT intouchable", + "check_optin": "", + "diff": [ + "CUDA_GRAPH_OPT : (absent) → on" + ], + "name": "graphes" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TOP_K": "20" + }, + "check": "clé TEMP intouchable", + "check_optin": "clé TEMP intouchable", + "diff": [ + "TEMP : 0.6 → (absent)" + ], + "name": "retrait" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "", + "check_optin": "", + "diff": [ + "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4" + ], + "name": "extra permis" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k f16 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "EXTRA_ARGS modifié hors liste blanche", + "check_optin": "EXTRA_ARGS modifié hors liste blanche", + "diff": [ + "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k f16 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4" + ], + "name": "extra interdit" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "", + "check_optin": "", + "diff": [ + "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4" + ], + "name": "extra placement" + }, + { + "base": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "cand": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --spec-draft-n-max 4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "check": "EXTRA_ARGS modifié hors liste blanche", + "check_optin": "", + "diff": [ + "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --spec-draft-n-max 4" + ], + "name": "spéculation dans extra" + }, + { + "base": { + "EXTRA_ARGS": "-t 8" + }, + "cand": {}, + "check": "", + "check_optin": "", + "diff": [ + "EXTRA_ARGS : -t 8 → (absent)" + ], + "name": "extra vide" + } + ], + "denature": [ + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "", + "denature_optin": "", + "name": "réglages permis", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "denature": "", + "denature_optin": "", + "name": "identique", + "same_key": true + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "", + "denature_optin": "", + "name": "argument de moins", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7", + "--jinja" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« --jinja » au lieu de « »)", + "denature_optin": "ligne de commande différente hors réglages permis (« --jinja » au lieu de « »)", + "name": "argument de plus", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "f16", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« f16 » au lieu de « q8_0 »)", + "denature_optin": "ligne de commande différente hors réglages permis (« f16 » au lieu de « q8_0 »)", + "name": "cache kv", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7", + "-cram", + "4096" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« -cram » au lieu de « »)", + "denature_optin": "ligne de commande différente hors réglages permis (« -cram » au lieu de « »)", + "name": "cache-ram", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "32768", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« 32768 » au lieu de « 65536 »)", + "denature_optin": "ligne de commande différente hors réglages permis (« 32768 » au lieu de « 65536 »)", + "name": "contexte", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "0,1", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "variable CUDA_VISIBLE_DEVICES différente (« 0,1 » au lieu de « 1,0 »)", + "denature_optin": "variable CUDA_VISIBLE_DEVICES différente (« 0,1 » au lieu de « 1,0 »)", + "name": "cuda", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "off", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« off » au lieu de « on »)", + "denature_optin": "ligne de commande différente hors réglages permis (« off » au lieu de « on »)", + "name": "fa", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_CUDA_GRAPH_OPT": "1", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "variable GGML_CUDA_GRAPH_OPT différente (« 1 » au lieu de « »)", + "denature_optin": "", + "name": "graphes", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7", + "-md", + "/d.gguf" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« -md » au lieu de « »)", + "denature_optin": "", + "name": "modèle ébauche", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« --cache-ram » au lieu de « -ot »)", + "denature_optin": "ligne de commande différente hors réglages permis (« --cache-ram » au lieu de « -ot »)", + "name": "ot", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7", + "--api-key", + "s3cret", + "--slot-save-path", + "/tmp/x", + "--port=9" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« --api-key » au lieu de « »)", + "denature_optin": "ligne de commande différente hors réglages permis (« --api-key » au lieu de « »)", + "name": "port et clé", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "2", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« 2 » au lieu de « 1 »)", + "denature_optin": "ligne de commande différente hors réglages permis (« 2 » au lieu de « 1 »)", + "name": "slots", + "same_key": false + }, + { + "base": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "2048", + "-ub", + "512", + "--host", + "127.0.0.1", + "--port", + "41000", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "--cache-ram", + "8192", + "--flash-attn", + "on" + ], + "env": { + "CUDA_SCALE_LAUNCH_QUEUES": "4x", + "CUDA_VISIBLE_DEVICES": "1,0" + } + }, + "cand": { + "bin": "", + "args": [ + "/bin/llama-server", + "-m", + "/m.gguf", + "-c", + "65536", + "-b", + "4096", + "-ub", + "2048", + "--host", + "127.0.0.1", + "--port", + "41001", + "--parallel", + "1", + "-ctk", + "q8_0", + "-ot", + "per_layer_token_embd=CPU", + "--cache-ram", + "12000", + "--flash-attn", + "on", + "-t", + "7", + "--spec-draft-n-max", + "4" + ], + "env": { + "CUDA_VISIBLE_DEVICES": "1,0", + "GGML_OP_OFFLOAD_MIN_BATCH": "128" + } + }, + "denature": "ligne de commande différente hors réglages permis (« --spec-draft-n-max » au lieu de « »)", + "denature_optin": "", + "name": "spéculation", + "same_key": false + } + ], + "extra": [ + { + "add_bs": "--backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "", + "drop_experts": "", + "drop_ts": "", + "drop_ub": "", + "in": "", + "removed": null, + "set_b": "-b 4096", + "set_t": "-t 7", + "set_t_space": "-t 'a b'", + "set_ub": "-ub 2048", + "spans": null, + "split": [] + }, + { + "add_bs": "--backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": " ", + "drop_experts": " ", + "drop_ts": " ", + "drop_ub": " ", + "in": " ", + "removed": null, + "set_b": "-b 4096", + "set_t": "-t 7", + "set_t_space": "-t 'a b'", + "set_ub": "-ub 2048", + "spans": null, + "split": [] + }, + { + "add_bs": "-b 2048 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-b 2048", + "drop_experts": "-b 2048", + "drop_ts": "-b 2048", + "drop_ub": "-b 2048", + "in": "-b 2048", + "removed": null, + "set_b": "-b 4096", + "set_t": "-b 2048 -t 7", + "set_t_space": "-b 2048 -t 'a b'", + "set_ub": "-b 2048 -ub 2048", + "spans": [ + [ + 0, + 2, + "-b" + ], + [ + 3, + 7, + "2048" + ] + ], + "split": [ + "-b", + "2048" + ] + }, + { + "add_bs": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 --backend-sampling", + "core": [ + "--chat-template-file", + "/mes modèles/tpl.jinja" + ], + "core_optin": [ + "--chat-template-file", + "/mes modèles/tpl.jinja" + ], + "drop_cmoe": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", + "drop_experts": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", + "drop_ts": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", + "drop_ub": "--chat-template-file \"/mes modèles/tpl.jinja\"", + "in": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", + "removed": null, + "set_b": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -b 4096", + "set_t": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -t 7", + "set_t_space": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -t 'a b'", + "set_ub": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 2048", + "spans": [ + [ + 0, + 20, + "--chat-template-file" + ], + [ + 21, + 46, + "/mes modèles/tpl.jinja" + ], + [ + 47, + 50, + "-ub" + ], + [ + 51, + 54, + "512" + ] + ], + "split": [ + "--chat-template-file", + "/mes modèles/tpl.jinja", + "-ub", + "512" + ] + }, + { + "add_bs": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", + "drop_experts": "", + "drop_ts": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", + "drop_ub": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", + "in": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", + "removed": [ + "-ot blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU", + "--n-cpu-moe 40" + ], + "set_b": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -b 4096", + "set_t": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -t 7", + "set_t_space": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -t 'a b'", + "set_ub": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -ub 2048", + "spans": [ + [ + 0, + 3, + "-ot" + ], + [ + 4, + 38, + "blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU" + ], + [ + 39, + 50, + "--n-cpu-moe" + ], + [ + 51, + 53, + "40" + ] + ], + "split": [ + "-ot", + "blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU", + "--n-cpu-moe", + "40" + ] + }, + { + "add_bs": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe --backend-sampling", + "core": [ + "-ot", + "per_layer_token_embd=CPU", + "--cache-type-k", + "q8_0", + "--flash-attn", + "on", + "--chat-template-file", + "/a b/t.jinja", + "-ctv", + "q8_0" + ], + "core_optin": [ + "-ot", + "per_layer_token_embd=CPU", + "--cache-type-k", + "q8_0", + "--flash-attn", + "on", + "--chat-template-file", + "/a b/t.jinja", + "-ctv", + "q8_0" + ], + "drop_cmoe": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0", + "drop_experts": "-ot per_layer_token_embd=CPU --cache-type-k q8_0 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0", + "drop_ts": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ctv q8_0 -cmoe", + "drop_ub": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", + "in": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", + "removed": [ + "-ot blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU", + "--n-cpu-moe 40", + "-cmoe" + ], + "set_b": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -b 4096", + "set_t": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -t 7", + "set_t_space": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -t 'a b'", + "set_ub": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 2048 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", + "spans": [ + [ + 0, + 3, + "-ot" + ], + [ + 4, + 63, + "blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU" + ], + [ + 64, + 78, + "--cache-type-k" + ], + [ + 79, + 83, + "q8_0" + ], + [ + 84, + 95, + "--n-cpu-moe" + ], + [ + 96, + 98, + "40" + ], + [ + 99, + 111, + "--flash-attn" + ], + [ + 112, + 114, + "on" + ], + [ + 115, + 118, + "-ub" + ], + [ + 119, + 122, + "512" + ], + [ + 123, + 143, + "--chat-template-file" + ], + [ + 144, + 158, + "/a b/t.jinja" + ], + [ + 159, + 162, + "-ts" + ], + [ + 163, + 170, + "0.9,0.1" + ], + [ + 171, + 175, + "-ctv" + ], + [ + 176, + 180, + "q8_0" + ], + [ + 181, + 186, + "-cmoe" + ] + ], + "split": [ + "-ot", + "blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU", + "--cache-type-k", + "q8_0", + "--n-cpu-moe", + "40", + "--flash-attn", + "on", + "-ub", + "512", + "--chat-template-file", + "/a b/t.jinja", + "-ts", + "0.9,0.1", + "-ctv", + "q8_0", + "-cmoe" + ] + }, + { + "add_bs": "-ub=1024 -t 8 -b 4096 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ub=1024 -t 8 -b 4096", + "drop_experts": "-ub=1024 -t 8 -b 4096", + "drop_ts": "-ub=1024 -t 8 -b 4096", + "drop_ub": "-t 8 -b 4096", + "in": "-ub=1024 -t 8 -b 4096", + "removed": null, + "set_b": "-ub=1024 -t 8 -b 4096", + "set_t": "-ub=1024 -t 7 -b 4096", + "set_t_space": "-ub=1024 -t 'a b' -b 4096", + "set_ub": "-ub=2048 -t 8 -b 4096", + "spans": [ + [ + 0, + 8, + "-ub=1024" + ], + [ + 9, + 11, + "-t" + ], + [ + 12, + 13, + "8" + ], + [ + 14, + 16, + "-b" + ], + [ + 17, + 21, + "4096" + ] + ], + "split": [ + "-ub=1024", + "-t", + "8", + "-b", + "4096" + ] + }, + { + "add_bs": " -ts 0.9,0.1 --flash-attn on --backend-sampling", + "core": [ + "--flash-attn", + "on" + ], + "core_optin": [ + "--flash-attn", + "on" + ], + "drop_cmoe": " -ts 0.9,0.1 --flash-attn on", + "drop_experts": " -ts 0.9,0.1 --flash-attn on", + "drop_ts": "--flash-attn on", + "drop_ub": " -ts 0.9,0.1 --flash-attn on", + "in": " -ts 0.9,0.1 --flash-attn on", + "removed": null, + "set_b": " -ts 0.9,0.1 --flash-attn on -b 4096", + "set_t": " -ts 0.9,0.1 --flash-attn on -t 7", + "set_t_space": " -ts 0.9,0.1 --flash-attn on -t 'a b'", + "set_ub": " -ts 0.9,0.1 --flash-attn on -ub 2048", + "spans": [ + [ + 2, + 5, + "-ts" + ], + [ + 6, + 13, + "0.9,0.1" + ], + [ + 15, + 27, + "--flash-attn" + ], + [ + 28, + 30, + "on" + ] + ], + "split": [ + "-ts", + "0.9,0.1", + "--flash-attn", + "on" + ] + }, + { + "add_bs": "-ts=1,1 -mg 0 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ts=1,1 -mg 0", + "drop_experts": "-ts=1,1 -mg 0", + "drop_ts": "-mg 0", + "drop_ub": "-ts=1,1 -mg 0", + "in": "-ts=1,1 -mg 0", + "removed": null, + "set_b": "-ts=1,1 -mg 0 -b 4096", + "set_t": "-ts=1,1 -mg 0 -t 7", + "set_t_space": "-ts=1,1 -mg 0 -t 'a b'", + "set_ub": "-ts=1,1 -mg 0 -ub 2048", + "spans": [ + [ + 0, + 7, + "-ts=1,1" + ], + [ + 8, + 11, + "-mg" + ], + [ + 12, + 13, + "0" + ] + ], + "split": [ + "-ts=1,1", + "-mg", + "0" + ] + }, + { + "add_bs": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 --backend-sampling", + "core": [ + "-ot", + "per_layer_token_embd=CPU" + ], + "core_optin": [ + "-ot", + "per_layer_token_embd=CPU" + ], + "drop_cmoe": "-ot exps=CPU,per_layer_token_embd=CPU -ncmoe=12", + "drop_experts": "-ot per_layer_token_embd=CPU", + "drop_ts": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", + "drop_ub": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", + "in": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", + "removed": [ + "-ot exps=CPU", + "-ncmoe 12", + "-cmoe", + "--cpu-moe" + ], + "set_b": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -b 4096", + "set_t": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -t 7", + "set_t_space": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -t 'a b'", + "set_ub": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -ub 2048", + "spans": [ + [ + 0, + 3, + "-ot" + ], + [ + 4, + 37, + "exps=CPU,per_layer_token_embd=CPU" + ], + [ + 38, + 43, + "-cmoe" + ], + [ + 44, + 53, + "--cpu-moe" + ], + [ + 54, + 63, + "-ncmoe=12" + ] + ], + "split": [ + "-ot", + "exps=CPU,per_layer_token_embd=CPU", + "-cmoe", + "--cpu-moe", + "-ncmoe=12" + ] + }, + { + "add_bs": "-ot=per_layer_token_embd=CPU --backend-sampling", + "core": [ + "-ot", + "per_layer_token_embd=CPU" + ], + "core_optin": [ + "-ot", + "per_layer_token_embd=CPU" + ], + "drop_cmoe": "-ot=per_layer_token_embd=CPU", + "drop_experts": "-ot=per_layer_token_embd=CPU", + "drop_ts": "-ot=per_layer_token_embd=CPU", + "drop_ub": "-ot=per_layer_token_embd=CPU", + "in": "-ot=per_layer_token_embd=CPU", + "removed": null, + "set_b": "-ot=per_layer_token_embd=CPU -b 4096", + "set_t": "-ot=per_layer_token_embd=CPU -t 7", + "set_t_space": "-ot=per_layer_token_embd=CPU -t 'a b'", + "set_ub": "-ot=per_layer_token_embd=CPU -ub 2048", + "spans": [ + [ + 0, + 28, + "-ot=per_layer_token_embd=CPU" + ] + ], + "split": [ + "-ot=per_layer_token_embd=CPU" + ] + }, + { + "add_bs": "--override-tensor exps=CPU --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "--override-tensor exps=CPU", + "drop_experts": "", + "drop_ts": "--override-tensor exps=CPU", + "drop_ub": "--override-tensor exps=CPU", + "in": "--override-tensor exps=CPU", + "removed": [ + "--override-tensor exps=CPU" + ], + "set_b": "--override-tensor exps=CPU -b 4096", + "set_t": "--override-tensor exps=CPU -t 7", + "set_t_space": "--override-tensor exps=CPU -t 'a b'", + "set_ub": "--override-tensor exps=CPU -ub 2048", + "spans": [ + [ + 0, + 17, + "--override-tensor" + ], + [ + 18, + 26, + "exps=CPU" + ] + ], + "split": [ + "--override-tensor", + "exps=CPU" + ] + }, + { + "add_bs": "-ot --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ot", + "drop_experts": "-ot", + "drop_ts": "-ot", + "drop_ub": "-ot", + "in": "-ot", + "removed": null, + "set_b": "-ot -b 4096", + "set_t": "-ot -t 7", + "set_t_space": "-ot -t 'a b'", + "set_ub": "-ot -ub 2048", + "spans": [ + [ + 0, + 3, + "-ot" + ] + ], + "split": [ + "-ot" + ] + }, + { + "add_bs": "-t --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-t", + "drop_experts": "-t", + "drop_ts": "-t", + "drop_ub": "-t", + "in": "-t", + "removed": null, + "set_b": "-t -b 4096", + "set_t": "-t 7", + "set_t_space": "-t 'a b'", + "set_ub": "-t -ub 2048", + "spans": [ + [ + 0, + 2, + "-t" + ] + ], + "split": [ + "-t" + ] + }, + { + "add_bs": "-t 8 -t 6 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-t 8 -t 6", + "drop_experts": "-t 8 -t 6", + "drop_ts": "-t 8 -t 6", + "drop_ub": "-t 8 -t 6", + "in": "-t 8 -t 6", + "removed": null, + "set_b": "-t 8 -t 6 -b 4096", + "set_t": "-t 8 -t 7", + "set_t_space": "-t 8 -t 'a b'", + "set_ub": "-t 8 -t 6 -ub 2048", + "spans": [ + [ + 0, + 2, + "-t" + ], + [ + 3, + 4, + "8" + ], + [ + 5, + 7, + "-t" + ], + [ + 8, + 9, + "6" + ] + ], + "split": [ + "-t", + "8", + "-t", + "6" + ] + }, + { + "add_bs": "-ub --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ub", + "drop_experts": "-ub", + "drop_ts": "-ub", + "drop_ub": "", + "in": "-ub", + "removed": null, + "set_b": "-ub -b 4096", + "set_t": "-ub -t 7", + "set_t_space": "-ub -t 'a b'", + "set_ub": "-ub 2048", + "spans": [ + [ + 0, + 3, + "-ub" + ] + ], + "split": [ + "-ub" + ] + }, + { + "add_bs": "\"\" -x --backend-sampling", + "core": [ + "", + "-x" + ], + "core_optin": [ + "", + "-x" + ], + "drop_cmoe": "\"\" -x", + "drop_experts": "\"\" -x", + "drop_ts": "\"\" -x", + "drop_ub": "\"\" -x", + "in": "\"\" -x", + "removed": null, + "set_b": "\"\" -x -b 4096", + "set_t": "\"\" -x -t 7", + "set_t_space": "\"\" -x -t 'a b'", + "set_ub": "\"\" -x -ub 2048", + "spans": [ + [ + 0, + 2, + "" + ], + [ + 3, + 5, + "-x" + ] + ], + "split": [ + "", + "-x" + ] + }, + { + "add_bs": "\"\"x y --backend-sampling", + "core": [ + "", + "x", + "y" + ], + "core_optin": [ + "", + "x", + "y" + ], + "drop_cmoe": "\"\"x y", + "drop_experts": "\"\"x y", + "drop_ts": "\"\"x y", + "drop_ub": "\"\"x y", + "in": "\"\"x y", + "removed": null, + "set_b": "\"\"x y -b 4096", + "set_t": "\"\"x y -t 7", + "set_t_space": "\"\"x y -t 'a b'", + "set_ub": "\"\"x y -ub 2048", + "spans": [ + [ + 0, + 2, + "" + ], + [ + 2, + 3, + "x" + ], + [ + 4, + 5, + "y" + ] + ], + "split": [ + "", + "x", + "y" + ] + }, + { + "add_bs": "a\"\"b --backend-sampling", + "core": [ + "ab" + ], + "core_optin": [ + "ab" + ], + "drop_cmoe": "a\"\"b", + "drop_experts": "a\"\"b", + "drop_ts": "a\"\"b", + "drop_ub": "a\"\"b", + "in": "a\"\"b", + "removed": null, + "set_b": "a\"\"b -b 4096", + "set_t": "a\"\"b -t 7", + "set_t_space": "a\"\"b -t 'a b'", + "set_ub": "a\"\"b -ub 2048", + "spans": [ + [ + 0, + 4, + "ab" + ] + ], + "split": [ + "ab" + ] + }, + { + "add_bs": "'a b'c \"d\" --backend-sampling", + "core": [ + "a bc", + "d" + ], + "core_optin": [ + "a bc", + "d" + ], + "drop_cmoe": "'a b'c \"d\"", + "drop_experts": "'a b'c \"d\"", + "drop_ts": "'a b'c \"d\"", + "drop_ub": "'a b'c \"d\"", + "in": "'a b'c \"d\"", + "removed": null, + "set_b": "'a b'c \"d\" -b 4096", + "set_t": "'a b'c \"d\" -t 7", + "set_t_space": "'a b'c \"d\" -t 'a b'", + "set_ub": "'a b'c \"d\" -ub 2048", + "spans": [ + [ + 0, + 6, + "a bc" + ], + [ + 7, + 10, + "d" + ] + ], + "split": [ + "a bc", + "d" + ] + }, + { + "add_bs": "--cache-type-k q8_0\t--flash-attn on\n-t 8 --backend-sampling", + "core": [ + "--cache-type-k", + "q8_0", + "--flash-attn", + "on" + ], + "core_optin": [ + "--cache-type-k", + "q8_0", + "--flash-attn", + "on" + ], + "drop_cmoe": "--cache-type-k q8_0\t--flash-attn on\n-t 8", + "drop_experts": "--cache-type-k q8_0\t--flash-attn on\n-t 8", + "drop_ts": "--cache-type-k q8_0\t--flash-attn on\n-t 8", + "drop_ub": "--cache-type-k q8_0\t--flash-attn on\n-t 8", + "in": "--cache-type-k q8_0\t--flash-attn on\n-t 8", + "removed": null, + "set_b": "--cache-type-k q8_0\t--flash-attn on\n-t 8 -b 4096", + "set_t": "--cache-type-k q8_0\t--flash-attn on\n-t 7", + "set_t_space": "--cache-type-k q8_0\t--flash-attn on\n-t 'a b'", + "set_ub": "--cache-type-k q8_0\t--flash-attn on\n-t 8 -ub 2048", + "spans": [ + [ + 0, + 14, + "--cache-type-k" + ], + [ + 15, + 19, + "q8_0" + ], + [ + 20, + 32, + "--flash-attn" + ], + [ + 33, + 35, + "on" + ], + [ + 36, + 38, + "-t" + ], + [ + 39, + 40, + "8" + ] + ], + "split": [ + "--cache-type-k", + "q8_0", + "--flash-attn", + "on", + "-t", + "8" + ] + }, + { + "add_bs": "\"non fermé -b 4 --backend-sampling", + "core": [ + "non fermé -b 4" + ], + "core_optin": [ + "non fermé -b 4" + ], + "drop_cmoe": "\"non fermé -b 4", + "drop_experts": "\"non fermé -b 4", + "drop_ts": "\"non fermé -b 4", + "drop_ub": "\"non fermé -b 4", + "in": "\"non fermé -b 4", + "removed": null, + "set_b": "\"non fermé -b 4 -b 4096", + "set_t": "\"non fermé -b 4 -t 7", + "set_t_space": "\"non fermé -b 4 -t 'a b'", + "set_ub": "\"non fermé -b 4 -ub 2048", + "spans": [ + [ + 0, + 16, + "non fermé -b 4" + ] + ], + "split": [ + "non fermé -b 4" + ] + }, + { + "add_bs": "x\"y z\"w -ub=1024 --backend-sampling", + "core": [ + "xy zw" + ], + "core_optin": [ + "xy zw" + ], + "drop_cmoe": "x\"y z\"w -ub=1024", + "drop_experts": "x\"y z\"w -ub=1024", + "drop_ts": "x\"y z\"w -ub=1024", + "drop_ub": "x\"y z\"w", + "in": "x\"y z\"w -ub=1024", + "removed": null, + "set_b": "x\"y z\"w -ub=1024 -b 4096", + "set_t": "x\"y z\"w -ub=1024 -t 7", + "set_t_space": "x\"y z\"w -ub=1024 -t 'a b'", + "set_ub": "x\"y z\"w -ub=2048", + "spans": [ + [ + 0, + 7, + "xy zw" + ], + [ + 8, + 16, + "-ub=1024" + ] + ], + "split": [ + "xy zw", + "-ub=1024" + ] + }, + { + "add_bs": "-bs --backend-sampling", + "core": [ + "-bs" + ], + "core_optin": [], + "drop_cmoe": "-bs", + "drop_experts": "-bs", + "drop_ts": "-bs", + "drop_ub": "-bs", + "in": "-bs", + "removed": null, + "set_b": "-bs -b 4096", + "set_t": "-bs -t 7", + "set_t_space": "-bs -t 'a b'", + "set_ub": "-bs -ub 2048", + "spans": [ + [ + 0, + 3, + "-bs" + ] + ], + "split": [ + "-bs" + ] + }, + { + "add_bs": "--backend-sampling -t 4 --backend-sampling", + "core": [ + "--backend-sampling" + ], + "core_optin": [], + "drop_cmoe": "--backend-sampling -t 4", + "drop_experts": "--backend-sampling -t 4", + "drop_ts": "--backend-sampling -t 4", + "drop_ub": "--backend-sampling -t 4", + "in": "--backend-sampling -t 4", + "removed": null, + "set_b": "--backend-sampling -t 4 -b 4096", + "set_t": "--backend-sampling -t 7", + "set_t_space": "--backend-sampling -t 'a b'", + "set_ub": "--backend-sampling -t 4 -ub 2048", + "spans": [ + [ + 0, + 18, + "--backend-sampling" + ], + [ + 19, + 21, + "-t" + ], + [ + 22, + 23, + "4" + ] + ], + "split": [ + "--backend-sampling", + "-t", + "4" + ] + }, + { + "add_bs": "-b 512\t-ub 256 --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-b 512\t-ub 256", + "drop_experts": "-b 512\t-ub 256", + "drop_ts": "-b 512\t-ub 256", + "drop_ub": "-b 512", + "in": "-b 512\t-ub 256", + "removed": null, + "set_b": "-b 4096\t-ub 256", + "set_t": "-b 512\t-ub 256 -t 7", + "set_t_space": "-b 512\t-ub 256 -t 'a b'", + "set_ub": "-b 512\t-ub 2048", + "spans": [ + [ + 0, + 2, + "-b" + ], + [ + 3, + 6, + "512" + ], + [ + 7, + 10, + "-ub" + ], + [ + 11, + 14, + "256" + ] + ], + "split": [ + "-b", + "512", + "-ub", + "256" + ] + }, + { + "add_bs": "-ncmoe --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "-ncmoe", + "drop_experts": "", + "drop_ts": "-ncmoe", + "drop_ub": "-ncmoe", + "in": "-ncmoe", + "removed": [ + "-ncmoe " + ], + "set_b": "-ncmoe -b 4096", + "set_t": "-ncmoe -t 7", + "set_t_space": "-ncmoe -t 'a b'", + "set_ub": "-ncmoe -ub 2048", + "spans": [ + [ + 0, + 6, + "-ncmoe" + ] + ], + "split": [ + "-ncmoe" + ] + }, + { + "add_bs": "--n-cpu-moe 0 --cpu-moe --backend-sampling", + "core": [], + "core_optin": [], + "drop_cmoe": "--n-cpu-moe 0", + "drop_experts": "", + "drop_ts": "--n-cpu-moe 0 --cpu-moe", + "drop_ub": "--n-cpu-moe 0 --cpu-moe", + "in": "--n-cpu-moe 0 --cpu-moe", + "removed": [ + "--n-cpu-moe 0", + "--cpu-moe" + ], + "set_b": "--n-cpu-moe 0 --cpu-moe -b 4096", + "set_t": "--n-cpu-moe 0 --cpu-moe -t 7", + "set_t_space": "--n-cpu-moe 0 --cpu-moe -t 'a b'", + "set_ub": "--n-cpu-moe 0 --cpu-moe -ub 2048", + "spans": [ + [ + 0, + 11, + "--n-cpu-moe" + ], + [ + 12, + 13, + "0" + ], + [ + 14, + 23, + "--cpu-moe" + ] + ], + "split": [ + "--n-cpu-moe", + "0", + "--cpu-moe" + ] + } + ], + "force_addr": [ + { + "in": [ + "/b", + "--host", + "0.0.0.0", + "--port", + "8080", + "-m", + "x" + ], + "out": [ + "/b", + "--host", + "127.0.0.1", + "--port", + "41234", + "-m", + "x" + ] + }, + { + "in": [ + "/b", + "--host=::", + "--port=1", + "--port", + "2", + "--host" + ], + "out": [ + "/b", + "--host=127.0.0.1", + "--port=41234", + "--port", + "41234", + "--host" + ] + }, + { + "in": [ + "/b", + "--hostname", + "x", + "-p", + "3" + ], + "out": [ + "/b", + "--hostname", + "x", + "-p", + "3" + ] + } + ], + "headroom": [ + { + "free": 0, + "gpus": null, + "ok": false, + "visible": null + }, + { + "free": 4576, + "gpus": [ + [ + 24576, + 20000 + ], + [ + 12288, + 1000 + ] + ], + "ok": true, + "visible": null + }, + { + "free": 11288, + "gpus": [ + [ + 24576, + 20000 + ], + [ + 12288, + 1000 + ] + ], + "ok": true, + "visible": [ + 1 + ] + }, + { + "free": -808, + "gpus": [ + [ + 0, + 0 + ], + [ + 8192, + 9000 + ] + ], + "ok": true, + "visible": null + }, + { + "free": 0, + "gpus": [ + [ + 8192, + 100 + ] + ], + "ok": false, + "visible": [ + 3 + ] + } + ], + "knobs": [ + { + "cfg": {}, + "get": { + "BATCH": "", + "FIT_TARGET": "", + "SPEC_N_MAX": "", + "THREADS": "", + "UBATCH": "" + }, + "set": { + "BATCH": { + "BATCH": "99" + }, + "FIT_TARGET": { + "FIT_TARGET": "99" + }, + "SPEC_N_MAX": { + "SPEC_N_MAX": "99" + }, + "THREADS": { + "THREADS": "99" + }, + "UBATCH": { + "UBATCH": "99" + } + }, + "unset": { + "BATCH": {}, + "FIT_TARGET": {}, + "SPEC_N_MAX": {}, + "THREADS": {}, + "UBATCH": {} + } + }, + { + "cfg": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "get": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "set": { + "BATCH": { + "BATCH": "99", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "FIT_TARGET": { + "BATCH": "4096", + "FIT_TARGET": "99", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "SPEC_N_MAX": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "99", + "THREADS": "0", + "UBATCH": "1024" + }, + "THREADS": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "99", + "UBATCH": "1024" + }, + "UBATCH": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "99" + } + }, + "unset": { + "BATCH": { + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "FIT_TARGET": { + "BATCH": "4096", + "SPEC_N_MAX": "2", + "THREADS": "0", + "UBATCH": "1024" + }, + "SPEC_N_MAX": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "THREADS": "0", + "UBATCH": "1024" + }, + "THREADS": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "UBATCH": "1024" + }, + "UBATCH": { + "BATCH": "4096", + "FIT_TARGET": "1536", + "SPEC_N_MAX": "2", + "THREADS": "0" + } + } + }, + { + "cfg": { + "EXTRA_ARGS": "-ub 512 -t 8", + "UBATCH": "1024" + }, + "get": { + "BATCH": "", + "FIT_TARGET": "", + "SPEC_N_MAX": "", + "THREADS": "8", + "UBATCH": "512" + }, + "set": { + "BATCH": { + "BATCH": "99", + "EXTRA_ARGS": "-ub 512 -t 8", + "UBATCH": "1024" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub 512 -t 8", + "FIT_TARGET": "99", + "UBATCH": "1024" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub 512 -t 8", + "SPEC_N_MAX": "99", + "UBATCH": "1024" + }, + "THREADS": { + "EXTRA_ARGS": "-ub 512 -t 99", + "UBATCH": "1024" + }, + "UBATCH": { + "EXTRA_ARGS": "-ub 99 -t 8", + "UBATCH": "1024" + } + }, + "unset": { + "BATCH": { + "EXTRA_ARGS": "-ub 512 -t 8", + "UBATCH": "1024" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub 512 -t 8", + "UBATCH": "1024" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub 512 -t 8", + "UBATCH": "1024" + }, + "THREADS": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": "1024" + }, + "UBATCH": { + "EXTRA_ARGS": "-t 8", + "UBATCH": "1024" + } + } + }, + { + "cfg": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" + }, + "get": { + "BATCH": "1024", + "FIT_TARGET": "2048", + "SPEC_N_MAX": "5", + "THREADS": "6", + "UBATCH": "256" + }, + "set": { + "BATCH": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 99 -fitt 2048 --spec-draft-n-max 5" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 99 --spec-draft-n-max 5" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 99" + }, + "THREADS": { + "EXTRA_ARGS": "-ub=256 --threads 99 -b 1024 -fitt 2048 --spec-draft-n-max 5" + }, + "UBATCH": { + "EXTRA_ARGS": "-ub=99 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" + } + }, + "unset": { + "BATCH": { + "EXTRA_ARGS": "-ub=256 --threads 6 -fitt 2048 --spec-draft-n-max 5" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 --spec-draft-n-max 5" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048" + }, + "THREADS": { + "EXTRA_ARGS": "-ub=256 -b 1024 -fitt 2048 --spec-draft-n-max 5" + }, + "UBATCH": { + "EXTRA_ARGS": "--threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" + } + } + }, + { + "cfg": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "get": { + "BATCH": "", + "FIT_TARGET": "", + "SPEC_N_MAX": "", + "THREADS": "", + "UBATCH": "512" + }, + "set": { + "BATCH": { + "BATCH": "99", + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub 512", + "FIT_TARGET": "99", + "UBATCH": " 128 " + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub 512", + "SPEC_N_MAX": "99", + "UBATCH": " 128 " + }, + "THREADS": { + "EXTRA_ARGS": "-ub 512", + "THREADS": "99", + "UBATCH": " 128 " + }, + "UBATCH": { + "EXTRA_ARGS": "-ub 99", + "UBATCH": " 128 " + } + }, + "unset": { + "BATCH": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "THREADS": { + "EXTRA_ARGS": "-ub 512", + "UBATCH": " 128 " + }, + "UBATCH": { + "UBATCH": " 128 " + } + } + }, + { + "cfg": { + "EXTRA_ARGS": "-t" + }, + "get": { + "BATCH": "", + "FIT_TARGET": "", + "SPEC_N_MAX": "", + "THREADS": "", + "UBATCH": "" + }, + "set": { + "BATCH": { + "BATCH": "99", + "EXTRA_ARGS": "-t" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-t", + "FIT_TARGET": "99" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-t", + "SPEC_N_MAX": "99" + }, + "THREADS": { + "EXTRA_ARGS": "-t 99" + }, + "UBATCH": { + "EXTRA_ARGS": "-t", + "UBATCH": "99" + } + }, + "unset": { + "BATCH": { + "EXTRA_ARGS": "-t" + }, + "FIT_TARGET": { + "EXTRA_ARGS": "-t" + }, + "SPEC_N_MAX": { + "EXTRA_ARGS": "-t" + }, + "THREADS": {}, + "UBATCH": { + "EXTRA_ARGS": "-t" + } + } + }, + { + "cfg": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "get": { + "BATCH": "", + "FIT_TARGET": "", + "SPEC_N_MAX": "", + "THREADS": "8", + "UBATCH": "512" + }, + "set": { + "BATCH": { + "BATCH": "99", + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "FIT_TARGET": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "FIT_TARGET": "99", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "SPEC_N_MAX": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "SPEC_N_MAX": "99", + "TEMP": "0.6", + "TOP_K": "20" + }, + "THREADS": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 99 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "UBATCH": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 99 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + } + }, + "unset": { + "BATCH": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "FIT_TARGET": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "SPEC_N_MAX": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "THREADS": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + }, + "UBATCH": { + "CTX": "65536", + "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -t 8 -ts 0.6,0.4", + "KV_TYPE": "q8_0", + "MMPROJ": "mmproj.gguf", + "MODEL": "big.gguf", + "PARALLEL": "1", + "REASONING": "on", + "REASONING_BUDGET": "-1", + "SPEC": "mtp", + "TEMP": "0.6", + "TOP_K": "20" + } + } + } + ], + "lock_owners": [ + { + "json": "{\"owner\":{\"pid\":42},\"via\":\"web\",\"phase\":\"essais\",\"since\":1700000000,\"main_was_active\":false}", + "key": "web|1700000000|0" + }, + { + "json": "{\"owner\":{\"pid\":42,\"start\":\"b0c1/123\",\"exes\":[\"/usr/bin/loki-rs\"]},\"via\":\"apply\",\"phase\":\"application\",\"since\":1700000000,\"nonce\":1700000000123456789,\"trial\":{\"pid\":43,\"start\":\"b0c1/456\",\"exes\":[\"/usr/bin/loki\",\"/opt/llama-server\"]},\"main_was_active\":true,\"backup\":\"/home/l/tune/backup/Dense-1.env\",\"preset_id\":\"Dense\",\"preset_name\":\"Dense\"}", + "key": "apply|1700000000|1700000000123456789" + } + ], + "logs": [ + { + "failure": "moteur arrêté : load_tensors: offloaded 49/49 layers to GPU", + "kv": "q8_0/q8_0", + "log": "load_tensors: offloaded 40/49 layers to GPU\nllama_kv_cache: size = 1024.00 MiB ( 8192 cells, 32 layers, 1/1 seqs), K (q8_0): 512.00 MiB, V (q8_0): 512.00 MiB\nload_tensors: offloaded 49/49 layers to GPU\n", + "offloaded": 49, + "total": 49 + }, + { + "failure": "moteur arrêté : rien", + "kv": "", + "log": "rien", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté au chargement", + "kv": "", + "log": "", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté : K (f16): x V (f16): y", + "kv": "f16/f16,q4_0/q8_0", + "log": "K (F16): 1 MiB, V (f16): 1 MiB\nK (q4_0): 1, V (q8_0): 2\nK (f16): x V (f16): y", + "offloaded": -1, + "total": -1 + }, + { + "failure": "mémoire insuffisante (OOM)", + "kv": "", + "log": "ggml_backend_cuda_buffer_type_alloc_buffer: cudaMalloc failed: out of memory", + "offloaded": -1, + "total": -1 + }, + { + "failure": "--fit n'a pas su placer le modèle", + "kv": "", + "log": "common_fit: failed to fit model\n", + "offloaded": -1, + "total": -1 + }, + { + "failure": "échec du chargement du modèle", + "kv": "", + "log": "llama_model_load: error loading model: bad magic\n", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté : derniere ligne de journal", + "kv": "", + "log": "ligne 1\nderniere ligne de journal \n\n", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté au chargement", + "kv": "", + "log": "\n \n", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté : éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "kv": "", + "log": "x\néééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "offloaded": -1, + "total": -1 + }, + { + "failure": "moteur arrêté : aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\ufffd", + "kv": "", + "log": "x\naaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaébbbbbbbbbbbbbbbbbbbb", + "offloaded": -1, + "total": -1 + } + ], + "patch": { + "content": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\n", + "cases": [ + { + "error": "", + "name": "threads retirés, lots", + "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=2048\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on\"\nBATCH=4096\n", + "win": { + "BATCH": "4096", + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on", + "MODEL": "big.gguf", + "UBATCH": "2048" + } + }, + { + "error": "clé CTX intouchable", + "name": "contexte", + "out": "", + "win": { + "CTX": "32768", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", + "MODEL": "big.gguf", + "UBATCH": "512" + } + }, + { + "error": "", + "name": "rien", + "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\n", + "win": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", + "MODEL": "big.gguf", + "UBATCH": "512" + } + }, + { + "error": "", + "name": "opt-in", + "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\nSPEC=auto\n", + "win": { + "CTX": "65536", + "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", + "MODEL": "big.gguf", + "SPEC": "auto", + "UBATCH": "512" + } + }, + { + "error": "", + "name": "retrait d'EXTRA_ARGS", + "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\n", + "win": { + "CTX": "65536", + "MODEL": "big.gguf", + "UBATCH": "512" + } + } + ] + }, + "results": [ + "{\"preset_id\":\"\",\"preset_name\":\"\",\"base_fp\":\"\",\"preset_fp\":\"\",\"engine\":\"\",\"engine_build\":0,\"weights\":{\"k\":0,\"g\":0,\"f_cold\":0,\"from\":\"\"},\"depth\":0,\"n_ctx\":0,\"slots\":0,\"baseline\":{\"stage\":\"\",\"label\":\"\",\"status\":\"\",\"gain\":0,\"headroom_mib\":0,\"sec\":0},\"trials\":null,\"gain\":0,\"placement\":false,\"opt_in\":false,\"at\":0}", + "{\"preset_id\":\"Dense\",\"preset_name\":\"Dense\",\"base_fp\":\"abc\",\"preset_fp\":\"def\",\"engine\":\"b7000-x\",\"engine_build\":7000,\"weights\":{\"k\":2000,\"g\":600,\"f_cold\":0.05,\"from\":\"défaut\"},\"depth\":8192,\"n_ctx\":16384,\"slots\":1,\"baseline\":{\"stage\":\"référence\",\"label\":\"configuration actuelle\",\"status\":\"mesuré\",\"runs\":[{\"turn_sec\":3.25,\"cold_pp\":812.5,\"cached_pp\":1500,\"tg\":42.75,\"prose_pp\":2000,\"prose_tg\":45.5,\"depth\":8192},{\"turn_sec\":3.5,\"prose_pp\":1900,\"prose_tg\":44}],\"turn_sec\":3.375,\"gain\":0,\"headroom_mib\":-1,\"offloaded\":\"33/33\",\"sec\":61.5},\"trials\":[{\"stage\":\"lots\",\"label\":\"UBATCH=512 BATCH=1024\",\"status\":\"retenu\",\"why\":\"−7.3 % par tour (seuil 3.0 %)\",\"runs\":[{\"turn_sec\":3.25,\"cold_pp\":812.5,\"cached_pp\":1500,\"tg\":42.75,\"prose_pp\":2000,\"prose_tg\":45.5,\"depth\":8192}],\"turn_sec\":3.125,\"gain\":0.0734,\"headroom_mib\":512,\"changes\":[\"UBATCH 2048 → 512\"],\"sec\":30},{\"stage\":\"lots\",\"label\":\"UBATCH=2048 BATCH=4096\",\"status\":\"écarté\",\"why\":\"dénature : x\",\"gain\":0,\"headroom_mib\":-1,\"sec\":0}],\"best\":\"UBATCH=512 BATCH=1024\",\"gain\":0.0734,\"set\":{\"BATCH\":\"1024\",\"UBATCH\":\"512\"},\"old\":{\"BATCH\":\"\",\"UBATCH\":\"2048\"},\"changes\":[\"UBATCH 2048 → 512\",\"BATCH + 1024\"],\"placement\":true,\"partial\":\"annulé\",\"skipped\":[\"marges : une seule carte\"],\"notes\":[\"n\"],\"opt_in\":true,\"at\":1700000000}" + ], + "runs": [ + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "", + "name": "complet", + "run": { + "turn_sec": 18.997317738791423, + "cold_pp": 800, + "cached_pp": 1026, + "tg": 40, + "prose_pp": 1000, + "prose_tg": 50, + "depth": 32768 + } + }, + { + "bench": null, + "error": "mesure vide", + "name": "vide", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0, + "elapsed_sec": 0 + }, + "error": "mesure vide", + "name": "sans decode", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0 + }, + "error": "", + "name": "rapide", + "run": { + "turn_sec": 14, + "prose_pp": 1000, + "prose_tg": 50 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 0, + "skipped": "petit", + "reuse": 0 + } + }, + "error": "", + "name": "profondeur sautée", + "run": { + "turn_sec": 14, + "prose_pp": 1000, + "prose_tg": 50 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0, + "partial": "contexte plein" + } + }, + "error": "mesure partielle : contexte plein", + "name": "partiel", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "mesure en profondeur incomplète", + "name": "sans froid", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "mesure en profondeur incomplète", + "name": "froid nul", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "mesure en profondeur incomplète", + "name": "sans tours", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "reuse": 0 + } + }, + "error": "mesure en profondeur incomplète", + "name": "sans decode à D", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": -1, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "", + "name": "cache inconnu", + "run": { + "turn_sec": 19.050002002002003, + "cold_pp": 800, + "cached_pp": 999, + "tg": 40, + "prose_pp": 1000, + "prose_tg": 50, + "depth": 32768 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 2100, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 2100, + "cached": -1, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "prefill des tours illisible", + "name": "cache inconnu, rien d'autre", + "run": { + "turn_sec": 0, + "prose_pp": 0, + "prose_tg": 0 + } + }, + { + "bench": { + "prompt_n": 0, + "prompt_ms": 0, + "prompt_per_second": 1000, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 50, + "elapsed_sec": 0, + "depth": { + "ctx": 0, + "target": 32768, + "cold": { + "new": 0, + "cached": 0, + "expected": 0, + "prompt_ms": 0, + "prompt_per_second": 800, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + "turns": [ + { + "new": 48, + "cached": 30000, + "expected": 30048, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + }, + { + "new": 48, + "cached": 32300, + "expected": 32348, + "prompt_ms": 2000, + "prompt_per_second": 0, + "predicted_n": 0, + "predicted_ms": 0, + "predicted_per_second": 0 + } + ], + "cached_per_second": 999, + "decode_per_second": 40, + "reuse": 0 + } + }, + "error": "", + "name": "rien de neuf", + "run": { + "turn_sec": 19.050002002002003, + "cold_pp": 800, + "cached_pp": 999, + "tg": 40, + "prose_pp": 1000, + "prose_tg": 50, + "depth": 32768 + } + } + ], + "same_exe": [ + { + "a": "/usr/local/bin/loki", + "b": "/usr/local/bin/loki (deleted)", + "same": true + }, + { + "a": "/usr/local/bin/loki", + "b": "/usr/bin/llama-server", + "same": false + }, + { + "a": "C:\\Loki\\loki.exe", + "b": "LOKI", + "same": true + }, + { + "a": "", + "b": "loki", + "same": false + }, + { + "a": "/a/llama-server", + "b": "llama-server", + "same": true + }, + { + "a": "/a/b/", + "b": "b", + "same": true + } + ], + "stage_wanted": [ + { + "name": "placement", + "only": null, + "wanted": true + }, + { + "name": "placement", + "only": [ + "lots" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "Délestage" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "files" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "FILES CUDA" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "marges-x" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + " threads " + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "opt" + ], + "wanted": true + }, + { + "name": "placement", + "only": [ + "x" + ], + "wanted": true + }, + { + "name": "marges", + "only": null, + "wanted": true + }, + { + "name": "marges", + "only": [ + "lots" + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "Délestage" + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "files" + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "FILES CUDA" + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "marges-x" + ], + "wanted": true + }, + { + "name": "marges", + "only": [ + " threads " + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "opt" + ], + "wanted": false + }, + { + "name": "marges", + "only": [ + "x" + ], + "wanted": false + }, + { + "name": "lots", + "only": null, + "wanted": true + }, + { + "name": "lots", + "only": [ + "lots" + ], + "wanted": true + }, + { + "name": "lots", + "only": [ + "Délestage" + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + "files" + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + "FILES CUDA" + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + "marges-x" + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + " threads " + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + "opt" + ], + "wanted": false + }, + { + "name": "lots", + "only": [ + "x" + ], + "wanted": false + }, + { + "name": "délestage", + "only": null, + "wanted": true + }, + { + "name": "délestage", + "only": [ + "lots" + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + "Délestage" + ], + "wanted": true + }, + { + "name": "délestage", + "only": [ + "files" + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + "FILES CUDA" + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + "marges-x" + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + " threads " + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + "opt" + ], + "wanted": false + }, + { + "name": "délestage", + "only": [ + "x" + ], + "wanted": false + }, + { + "name": "threads", + "only": null, + "wanted": true + }, + { + "name": "threads", + "only": [ + "lots" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + "Délestage" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + "files" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + "FILES CUDA" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + "marges-x" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + " threads " + ], + "wanted": true + }, + { + "name": "threads", + "only": [ + "opt" + ], + "wanted": false + }, + { + "name": "threads", + "only": [ + "x" + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": null, + "wanted": true + }, + { + "name": "files CUDA", + "only": [ + "lots" + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": [ + "Délestage" + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": [ + "files" + ], + "wanted": true + }, + { + "name": "files CUDA", + "only": [ + "FILES CUDA" + ], + "wanted": true + }, + { + "name": "files CUDA", + "only": [ + "marges-x" + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": [ + " threads " + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": [ + "opt" + ], + "wanted": false + }, + { + "name": "files CUDA", + "only": [ + "x" + ], + "wanted": false + }, + { + "name": "opt-in", + "only": null, + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "lots" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "Délestage" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "files" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "FILES CUDA" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "marges-x" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + " threads " + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "opt" + ], + "wanted": true + }, + { + "name": "opt-in", + "only": [ + "x" + ], + "wanted": true + } + ], + "trial_config": [ + { + "cfg": { + "EXTRA_ARGS": "-ot \"a b\" -t 8", + "MODEL": "m.gguf", + "PORT": "41000" + }, + "error": "", + "text": "EXTRA_ARGS=-ot \"a b\" -t 8\nMODEL=m.gguf\nPORT=41000\n" + }, + { + "cfg": { + "MODEL": "x\ny" + }, + "error": "valeur de MODEL non représentable dans un fichier de configuration", + "text": "" + }, + { + "cfg": { + "A": " b ", + "NAME": "é ü" + }, + "error": "", + "text": "A=\" b \"\nNAME=\"é ü\"\n" + }, + { + "cfg": { + "A": "\"q\"", + "B": "c'd", + "C": "#h" + }, + "error": "valeur de A non représentable dans un fichier de configuration", + "text": "" + } + ], + "visible": [ + { + "in": "", + "out": null + }, + { + "in": "0", + "out": [ + 0 + ] + }, + { + "in": "1,0", + "out": [ + 1, + 0 + ] + }, + { + "in": " 2 , 3 ", + "out": [ + 2, + 3 + ] + }, + { + "in": "GPU-abc", + "out": null + }, + { + "in": "0,,1", + "out": null + }, + { + "in": "+1,-2", + "out": [ + 1, + -2 + ] + } + ], + "weights": [ + { + "name": "vide", + "recs": null, + "weights": { + "k": 2000, + "g": 600, + "f_cold": 0.05, + "from": "défaut" + } + }, + { + "name": "trop peu", + "recs": [ + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1000, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1001, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1002, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1003, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1004, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1005, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1006, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1007, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1008, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1009, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1010, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1011, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1012, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1013, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1014, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1015, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1016, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1017, + "gen": 300 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "new": 1018, + "gen": 300 + } + ], + "weights": { + "k": 2000, + "g": 600, + "f_cold": 0.05, + "from": "défaut" + } + }, + { + "name": "télémétrie", + "recs": [ + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4000, + "cached": 100, + "new": 500, + "gen": 100 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4100, + "cached": 3900, + "new": 537, + "gen": 113 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4200, + "cached": 4000, + "new": 574, + "gen": 126 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4300, + "cached": 100, + "new": 611, + "gen": 139 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4400, + "cached": 4200, + "new": 648, + "gen": 152 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4500, + "cached": 4300, + "new": 685, + "gen": 165 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4600, + "cached": 100, + "new": 722, + "gen": 178 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4700, + "cached": 4500, + "new": 759, + "gen": 191 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4800, + "cached": 4600, + "new": 796, + "gen": 204 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 4900, + "cached": 100, + "new": 833, + "gen": 217 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5000, + "cached": 4800, + "new": 870, + "gen": 230 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5100, + "cached": 4900, + "new": 907, + "gen": 243 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5200, + "cached": 100, + "new": 944, + "gen": 256 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5300, + "cached": 5100, + "new": 981, + "gen": 269 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5400, + "cached": 5200, + "new": 1018, + "gen": 282 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5500, + "cached": 100, + "new": 1055, + "gen": 295 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5600, + "cached": 5400, + "new": 1092, + "gen": 308 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5700, + "cached": 5500, + "new": 1129, + "gen": 321 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5800, + "cached": 100, + "new": 1166, + "gen": 334 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 5900, + "cached": 5700, + "new": 1203, + "gen": 347 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6000, + "cached": 5800, + "new": 1240, + "gen": 360 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6100, + "cached": 100, + "new": 1277, + "gen": 373 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6200, + "cached": 6000, + "new": 1314, + "gen": 386 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6300, + "cached": 6100, + "new": 1351, + "gen": 399 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6400, + "cached": 100, + "new": 1388, + "gen": 412 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6500, + "cached": 6300, + "new": 1425, + "gen": 425 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6600, + "cached": 6400, + "new": 1462, + "gen": 438 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6700, + "cached": 100, + "new": 1499, + "gen": 451 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6800, + "cached": 6600, + "new": 1536, + "gen": 464 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 6900, + "cached": 6700, + "new": 1573, + "gen": 477 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "bench", + "iter": 0, + "complete": true, + "new": 99999, + "gen": 99999 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": false, + "new": 99999, + "gen": 99999 + } + ], + "weights": { + "k": 1036.5, + "g": 288.5, + "f_cold": 0.3333333333333333, + "from": "télémétrie" + } + }, + { + "name": "bornes", + "recs": [ + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "task", + "iter": 0, + "complete": true, + "total": 100, + "cached": 0, + "new": 50, + "gen": 9000 + } + ], + "weights": { + "k": 128, + "g": 8192, + "f_cold": 0.05, + "from": "télémétrie" + } + }, + { + "name": "pair, froid connu", + "recs": [ + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1000, + "gen": 200 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1010, + "gen": 201 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1020, + "gen": 202 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1030, + "gen": 203 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1040, + "gen": 204 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1050, + "gen": 205 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1060, + "gen": 206 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1070, + "gen": 207 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1080, + "gen": 208 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1090, + "gen": 209 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1100, + "gen": 210 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1110, + "gen": 211 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1120, + "gen": 212 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1130, + "gen": 213 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1140, + "gen": 214 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1150, + "gen": 215 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1160, + "gen": 216 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1170, + "gen": 217 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1180, + "gen": 218 + }, + { + "at": "0001-01-01T00:00:00Z", + "seq": 0, + "kind": "main", + "iter": 0, + "complete": true, + "total": 2048, + "cached": 204, + "new": 1190, + "gen": 219 + } + ], + "weights": { + "k": 1095, + "g": 209.5, + "f_cold": 1, + "from": "télémétrie" + } + } + ], + "wins": [ + { + "base": [ + 20, + 20.4 + ], + "cand": [ + 19.7, + 19.8 + ], + "gain": 0.02227722772277224, + "mean_cand": 19.75, + "need": 0.03, + "spread_base": 0.019801980198019733, + "wins": false + }, + { + "base": [ + 20, + 20.4 + ], + "cand": [ + 18 + ], + "gain": 0.10891089108910888, + "mean_cand": 18, + "need": 0.03, + "spread_base": 0.019801980198019733, + "wins": false + }, + { + "base": [ + 20, + 20.4 + ], + "cand": [ + 18, + 18.1 + ], + "gain": 0.10643564356435636, + "mean_cand": 18.05, + "need": 0.03, + "spread_base": 0.019801980198019733, + "wins": true + }, + { + "base": [ + 18, + 22 + ], + "cand": [ + 18.5, + 18.6 + ], + "gain": 0.07249999999999997, + "mean_cand": 18.55, + "need": 0.2, + "spread_base": 0.2, + "wins": false + }, + { + "base": [], + "cand": [ + 1, + 2 + ], + "gain": 0, + "mean_cand": 1.5, + "need": 0.03, + "spread_base": 0, + "wins": false + }, + { + "base": [ + 10 + ], + "cand": [], + "gain": 0, + "mean_cand": 0, + "need": 0.03, + "spread_base": 0, + "wins": false + }, + { + "base": [ + 10, + 10 + ], + "cand": [ + 9.6, + 9.6 + ], + "gain": 0.040000000000000036, + "mean_cand": 9.6, + "need": 0.03, + "spread_base": 0, + "wins": true + }, + { + "base": [ + 10, + 10 + ], + "cand": [ + 9.7, + 9.7 + ], + "gain": 0.030000000000000072, + "mean_cand": 9.7, + "need": 0.03, + "spread_base": 0, + "wins": true + }, + { + "base": [ + 0 + ], + "cand": [ + 1, + 1 + ], + "gain": 0, + "mean_cand": 1, + "need": 0.03, + "spread_base": 0, + "wins": false + }, + { + "base": [ + 10, + 12 + ], + "cand": [ + 8, + 8.2, + 8.1 + ], + "gain": 0.26363636363636367, + "mean_cand": 8.1, + "need": 0.18181818181818182, + "spread_base": 0.18181818181818182, + "wins": true + } + ], + "worth_repeat": [ + { + "base": [ + 20, + 20.4 + ], + "first": 19, + "worth": true + }, + { + "base": [ + 20, + 20.4 + ], + "first": 19.8, + "worth": false + }, + { + "base": [ + 20 + ], + "first": 19.39, + "worth": true + }, + { + "base": [ + 20 + ], + "first": 19.41, + "worth": false + }, + { + "base": null, + "first": 1, + "worth": false + } + ] +} diff --git a/rust/loki-rs/tests/proxy.rs b/rust/loki-rs/tests/proxy.rs index 3a639ac..a5aad00 100644 --- a/rust/loki-rs/tests/proxy.rs +++ b/rust/loki-rs/tests/proxy.rs @@ -117,6 +117,7 @@ async fn facade(backend: SocketAddr) -> (SocketAddr, oneshot::Sender<()>) { strata: None, tasks: None, bench: None, + tune: None, }; tokio::spawn(loki_rs::proxy::serve(l, cfg, async move { let _ = rx.await; diff --git a/rust/loki-rs/tests/tune.rs b/rust/loki-rs/tests/tune.rs new file mode 100644 index 0000000..75114e9 --- /dev/null +++ b/rust/loki-rs/tests/tune.rs @@ -0,0 +1,821 @@ +//! Lot 28 : l'optimiseur tenu par la façade (`LOKI_TUNE=rust`). +//! +//! Les essais passent par de vrais processus : ce binaire de test joue +//! « loki serve » (la seule fonction dont le nom contient « serve », +//! sélectionnée par l'argument que la façade passe). À blanc, il écrit la +//! ligne de commande de l'essai ; pour de vrai, il sert un faux llama-server +//! dont le prefill est d'autant plus rapide que le micro-lot est petit. + +mod common; + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use common::fakego; +use loki_rs::tune::job::Tune; +use serde_json::{Value, json}; + +const FAUX: &str = "LOKI_TUNE_FAUX"; +const BUSY: &str = "optimisation en cours (loki tune) : le moteur est réservé aux essais — attends la fin ou annule-la"; + +// --------------------------------------------------------------------------- +// Le faux moteur +// --------------------------------------------------------------------------- + +/// Ce que le faux moteur annonce : son contexte, et s'il travaille déjà. +#[derive(Default)] +struct Knobs { + n_ctx: Mutex, + busy: Mutex, +} + +/// Un faux llama-server (fil à part, jusqu'à la fin du processus). +fn fake_llama(listener: TcpListener, ubatch: i64, knobs: Arc) { + let last = Arc::new(Mutex::new(0i64)); + std::thread::spawn(move || { + for st in listener.incoming().flatten() { + let (last, knobs) = (last.clone(), knobs.clone()); + std::thread::spawn(move || { + let _ = answer(st, ubatch, &knobs, &last); + }); + } + }); +} + +fn answer( + st: std::net::TcpStream, + ubatch: i64, + knobs: &Knobs, + last: &Mutex, +) -> std::io::Result<()> { + let n_ctx = *knobs.n_ctx.lock().unwrap(); + let mut r = BufReader::new(st.try_clone()?); + let mut line = String::new(); + r.read_line(&mut line)?; + let path = line.split(' ').nth(1).unwrap_or("").to_string(); + let mut len = 0usize; + loop { + let mut h = String::new(); + r.read_line(&mut h)?; + if h.trim().is_empty() { + break; + } + if let Some((k, v)) = h.split_once(':') + && k.eq_ignore_ascii_case("content-length") + { + len = v.trim().parse().unwrap_or(0); + } + } + let mut body = vec![0u8; len]; + r.read_exact(&mut body)?; + let req: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + let out = match path.as_str() { + "/health" => json!({ "status": "ok" }), + "/props" => json!({ + "n_ctx": n_ctx, "total_slots": 1, "build_info": "b7000-faux", + "default_generation_settings": { "n_ctx": n_ctx }, + }), + "/slots" => json!([{ "id": 0, "is_processing": *knobs.busy.lock().unwrap() }]), + "/tokenize" => { + let n = req["content"].as_str().unwrap_or("").len() / 4; + json!({ "tokens": vec![0; n.max(1)] }) + } + "/v1/chat/completions" => { + let chars: usize = req["messages"] + .as_array() + .map(|m| { + m.iter() + .map(|x| x["content"].as_str().unwrap_or("").len()) + .sum() + }) + .unwrap_or(0); + let total = (chars / 4).max(1) as i64; + let mut l = last.lock().unwrap(); + let cached = if req["cache_prompt"] == json!(true) && *l <= total { + *l + } else { + 0 + }; + *l = total; + let new = total - cached; + // Le prefill ralentit avec le micro-lot ; le decode, non. + let pps = 4_000_000.0 / ubatch as f64; + let n_gen = req["max_tokens"].as_i64().unwrap_or(16); + json!({ + "choices": [{ "message": { "content": "ok" } }], + "usage": { "prompt_tokens": total }, + "timings": { + "prompt_n": new, "prompt_ms": (new as f64 / pps * 1000.0).max(1.0), + "predicted_n": n_gen, "predicted_ms": n_gen as f64 / 50.0 * 1000.0, + "cache_n": cached, + }, + }) + } + _ => { + let mut w = st; + return w.write_all( + b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + } + }; + let b = out.to_string(); + let mut w = st; + write!( + w, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{b}", + b.len() + ) +} + +/// « loki serve » pour l'optimiseur : ce binaire relancé par la façade avec +/// `LOKI_TUNE_FAUX`. Ailleurs : rien. +#[test] +fn serve() { + if std::env::var(FAUX).is_err() { + return; + } + let dir = PathBuf::from(std::env::var("LOKI_TUNE_TRIAL").unwrap()); + let cfg = + loki_rs::presets::parse_env(&std::fs::read_to_string(dir.join("config.env")).unwrap()); + let get = |k: &str| cfg.get(k).cloned().unwrap_or_default(); + let ub = get("UBATCH"); + let die = |msg: &str| -> ! { + let _ = std::io::stderr().write_all(format!("{msg}\n").as_bytes()); + std::process::exit(1); + }; + if std::env::var("LOKI_TUNE_DRYRUN").as_deref() == Ok("1") { + if get("REFUS") == ub { + die(&format!("\x1b[31m[err]\x1b[0m UBATCH={ub} refusé")); + } + let mut args = vec!["-m".to_string(), get("MODEL"), "-c".into(), get("CTX")]; + for (k, f) in [("UBATCH", "-ub"), ("BATCH", "-b")] { + if !get(k).is_empty() { + args.extend([f.to_string(), get(k)]); + } + } + args.extend(["--host".into(), get("HOST"), "--port".into(), get("PORT")]); + let launch = json!({ + "bin": "/opt/faux/llama-server", "args": args, + "probe": { "help": "", "gpus": 0, "vram_mib": 0, "model_bytes": 1000, "moe": false, "cpu_threads": 8 }, + }); + std::fs::write(dir.join("launch.json"), launch.to_string()).unwrap(); + return; + } + if get("ECHEC") == ub { + die("llama_model_load: error loading model: out of memory"); + } + let port: u16 = get("PORT").parse().unwrap(); + let l = TcpListener::bind(("127.0.0.1", port)).unwrap(); + let _ = std::io::stdout().write_all(b"load_tensors: offloaded 33/33 layers to GPU\n"); + let knobs = Knobs::default(); + *knobs.n_ctx.lock().unwrap() = get("CTX").parse().unwrap(); + fake_llama(l, ub.parse().unwrap_or(2048), Arc::new(knobs)); + // Arrêté par la façade ; un essai oublié (test interrompu) s'en va seul. + std::thread::sleep(Duration::from_secs(600)); +} + +// --------------------------------------------------------------------------- +// Le banc +// --------------------------------------------------------------------------- + +struct Tn { + facade: std::net::SocketAddr, + go: Arc, + _stop: tokio::sync::oneshot::Sender<()>, + home: PathBuf, + port: u16, + main: Arc, + _tmp: tempfile::TempDir, +} + +const PRESET: &str = "# NAME=Dense\nCTX=32768\nUBATCH=2048\nREASONING=off\nCACHE_ISOLATE=off\n"; + +async fn start() -> Tn { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().to_path_buf(); + let mut t = Tune::new(home.clone(), std::env::current_exe().unwrap()); + t.trial_env = vec![(FAUX.into(), "1".into())]; + t.poll = Duration::from_millis(20); + t.probe_poll = Duration::from_millis(20); + t.settle = Duration::ZERO; + t.load_timeout = Duration::from_secs(20); + t.lease_ttl = Duration::from_millis(300); + let tune = Arc::new(t); + // Moteur et modèle présents : la relance du vrai moteur passe ses + // vérifications. + std::fs::write(home.join("llama-server"), "x").unwrap(); + std::fs::write(home.join("m.gguf"), "x").unwrap(); + let (facade, go, stop, _) = fakego::start_with( + home.clone(), + fakego::Opts { + tune: Some(tune), + ..Default::default() + }, + ) + .await; + // Le vrai moteur, en service. + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + let main: Arc = Arc::default(); + *main.n_ctx.lock().unwrap() = 32768; + fake_llama(l, 2048, main.clone()); + let tn = Tn { + facade, + go, + _stop: stop, + home, + port, + main, + _tmp: tmp, + }; + tn.preset(PRESET); + *tn.go.service_active.lock().unwrap() = true; + *tn.go.apply_presets.lock().unwrap() = true; + tn +} + +impl Tn { + /// Le preset « dense », et la configuration qui lui correspond (elle + /// seule). + fn preset(&self, content: &str) { + let dir = self.home.join("presets"); + std::fs::create_dir_all(&dir).unwrap(); + let h = self.home.display(); + let content = format!( + "{content}MODEL={h}/m.gguf\nBIN={h}/llama-server\nPORT={}\n", + self.port + ); + std::fs::write(dir.join("dense.env"), &content).unwrap(); + for k in self.go.config_map().keys() { + self.go.del("config", k); + } + for (k, v) in loki_rs::presets::parse_env(&content) { + self.go.set("config", &k, &v); + } + } + + async fn call(&self, method: &str, path: &str, body: &str) -> (u16, Value) { + let (code, b) = fakego::send(self.facade, method, path, &[], body).await; + let v = if b.is_empty() { + Value::Null + } else { + serde_json::from_slice(&b) + .unwrap_or_else(|_| panic!("illisible : {}", String::from_utf8_lossy(&b))) + }; + (code.as_u16(), v) + } + + async fn status(&self) -> Value { + self.call("GET", "/api/tune/status", "").await.1 + } + + async fn wait(&self, what: &str, key: &str) -> Value { + for _ in 0..6000 { + let st = self.status().await; + if st[key] != json!(true) { + return st; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{what} : ne finit pas"); + } + + fn config(&self) -> HashMap { + ["MODEL", "CTX", "UBATCH", "BATCH", "PORT"] + .iter() + .filter_map(|k| self.go.get("config", k).map(|v| (k.to_string(), v))) + .collect() + } +} + +fn trials(res: &Value) -> Vec<(String, String, String)> { + res["trials"] + .as_array() + .map(|a| { + a.iter() + .map(|t| { + ( + t["label"].as_str().unwrap_or("").to_string(), + t["status"].as_str().unwrap_or("").to_string(), + t["why"].as_str().unwrap_or("").to_string(), + ) + }) + .collect() + }) + .unwrap_or_default() +} + +// --------------------------------------------------------------------------- +// Les tests +// --------------------------------------------------------------------------- + +/// Une optimisation entière : vrai moteur arrêté puis relancé, essais +/// composés à blanc puis lancés, mesurés, retenus ; puis l'application dans +/// une copie, dans le preset, et le refus d'un preset changé depuis. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn optimisation_complete() { + let tn = start().await; + let (code, out) = tn + .call("POST", "/api/tune", r#"{"stages":["lots"],"budget_min":5}"#) + .await; + assert_eq!((code, out), (202, json!({ "ok": true }))); + assert_eq!(*tn.go.lease_whys.lock().unwrap(), vec!["tune".to_string()]); + // Pendant la mesure : le verrou de LOKI_HOME, au nom de la façade. + let mut seen_lock = false; + for _ in 0..2000 { + if let Some(o) = loki_rs::tune::lock::read_owner(&tn.home.join("tune.lock")) { + assert_eq!(o.owner.pid, i64::from(std::process::id())); + assert_eq!(o.via, "web"); + assert!(o.main_was_active); + seen_lock = true; + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!(seen_lock, "verrou jamais posé"); + let (code, out) = tn.call("POST", "/api/tune", "{}").await; + assert_eq!(code, 409); + assert_eq!(out["error"], json!(BUSY)); + let st = tn.wait("optimisation", "running").await; + assert_eq!(st["error"], Value::Null, "{st}"); + let res = &st["result"]; + assert_eq!(res["preset_id"], json!("dense")); + assert_eq!(res["preset_name"], json!("Dense")); + assert_eq!(res["engine"], json!("b7000-faux")); + assert_eq!(res["engine_build"], json!(7000)); + assert_eq!( + (res["n_ctx"].clone(), res["slots"].clone()), + (json!(32768), json!(1)) + ); + assert_eq!(res["depth"], json!(16384)); + assert_eq!(res["baseline"]["status"], json!("mesuré")); + assert_eq!(res["baseline"]["runs"].as_array().unwrap().len(), 2); + assert_eq!(res["baseline"]["offloaded"], json!("33/33")); + let tr = trials(res); + assert_eq!( + tr.iter().map(|t| t.0.as_str()).collect::>(), + [ + "UBATCH=512 BATCH=2048", + "UBATCH=512 BATCH=1024", + "UBATCH=1024 BATCH=2048", + "UBATCH=2048 BATCH=4096" + ] + ); + assert_eq!(tr[0].1, "retenu", "{tr:?}"); + assert!( + tr[0].2.starts_with('−') && tr[0].2.contains("par tour (seuil"), + "{tr:?}" + ); + assert_eq!( + res["best"], + json!(tr.iter().rfind(|t| t.1 == "retenu").unwrap().0) + ); + assert_eq!(res["set"]["UBATCH"], json!("512")); + assert!(res["gain"].as_f64().unwrap() > 0.03, "{res}"); + let skipped: Vec<&str> = res["skipped"] + .as_array() + .unwrap() + .iter() + .map(|s| s.as_str().unwrap()) + .collect(); + assert!(skipped.contains(&"marges : non demandé"), "{skipped:?}"); + assert!( + res["notes"] + .as_array() + .unwrap() + .contains(&json!("VRAM libre non mesurable (nvidia-smi absent) : ni micro-lot plus grand ni marges --fit essayés")) + ); + // Le vrai moteur : arrêté, puis relancé, sans le refus de l'optimiseur ; + // la dictée coupée ; le verrou rendu ; le bail aussi. + assert_eq!(*tn.go.service_os.lock().unwrap(), ["stop", "start"]); + assert_eq!(*tn.go.dictate_stops.lock().unwrap(), 1); + assert!(!tn.home.join("tune.lock").exists(), "verrou non rendu"); + assert!(tn.go.lease.lock().unwrap().is_none(), "bail non rendu"); + // Chaque essai a sa copie de configuration, sur une adresse privée ; + // config.env n'est jamais touché. + let e1 = loki_rs::presets::parse_env( + &std::fs::read_to_string(tn.home.join("tune/run/e01/config.env")).unwrap(), + ); + assert_eq!(e1.get("HOST").map(String::as_str), Some("127.0.0.1")); + assert_ne!(e1.get("PORT"), Some(&tn.port.to_string())); + assert_eq!(tn.config()["UBATCH"], "2048"); + // Enregistré par preset. + let saved: Value = serde_json::from_str(&tn.go.get("state", "tune_results").unwrap()).unwrap(); + assert_eq!(saved["dense"]["best"], res["best"]); + let (_, last) = tn.call("GET", "/api/tune/last?id=dense", "").await; + assert_eq!(last["ok"], json!(true)); + assert_eq!(last["preset_changed"], json!(false)); + assert_eq!(last["result"], *res); + + // Copie : rien d'autre ne bouge. + let (code, out) = tn + .call("POST", "/api/tune/apply", r#"{"target":"copy"}"#) + .await; + assert_eq!(code, 200, "{out}"); + let msg = out["message"].as_str().unwrap(); + assert!( + msg.starts_with("copie « Dense (optimisé) » créée ("), + "{msg}" + ); + assert!(msg.ends_with(") — bascule dessus pour l'utiliser"), "{msg}"); + assert!( + std::fs::read_to_string(tn.home.join("presets/dense.env")) + .unwrap() + .contains("UBATCH=2048") + ); + assert_eq!(*tn.go.service_os.lock().unwrap(), ["stop", "start"]); + + // Le preset lui-même : réécrit, appliqué, moteur redémarré et sondé. + let (code, out) = tn + .call("POST", "/api/tune/apply", r#"{"target":"preset"}"#) + .await; + assert_eq!((code, out), (202, json!({ "ok": true }))); + let st = tn.wait("application", "applying").await; + assert_eq!(st["apply"]["error"], json!(""), "{st}"); + assert_eq!( + st["apply"]["message"], + json!("preset « Dense » optimisé et vérifié") + ); + let log: Vec<&str> = st["apply"]["log"] + .as_array() + .unwrap() + .iter() + .map(|s| s.as_str().unwrap()) + .collect(); + assert!( + log[0].starts_with("écriture du preset (sauvegarde : "), + "{log:?}" + ); + assert_eq!( + &log[1..], + [ + "redémarrage du moteur", + "attente du moteur", + "sonde : prompt de 16384 jetons puis decode" + ] + ); + let preset = std::fs::read_to_string(tn.home.join("presets/dense.env")).unwrap(); + assert!( + preset.contains("UBATCH=512") && preset.contains("# NAME=Dense"), + "{preset}" + ); + assert_eq!(tn.config()["UBATCH"], "512"); + assert_eq!( + *tn.go.service_os.lock().unwrap(), + ["stop", "start", "restart"] + ); + assert!(!tn.home.join("tune.lock").exists(), "verrou non rendu"); + // Le preset a changé depuis la mesure : plus rien ne s'applique. + let (code, out) = tn + .call("POST", "/api/tune/apply", r#"{"target":"preset"}"#) + .await; + assert_eq!( + (code, out), + ( + 200, + json!({ "ok": false, "error": "le preset a changé depuis la mesure : relance l'optimisation" }) + ) + ); + let (_, last) = tn.call("GET", "/api/tune/last?id=dense", "").await; + assert_eq!(last["preset_changed"], json!(true)); +} + +/// Une sonde qui échoue après application : l'ancienne version revient — +/// preset ET configuration — et le moteur est redémarré dessus. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn application_defaite() { + let tn = start().await; + tn.call("POST", "/api/tune", r#"{"stages":["lots"]}"#).await; + let st = tn.wait("optimisation", "running").await; + assert_eq!(st["result"]["set"]["UBATCH"], json!("512"), "{st}"); + // Le moteur relancé ne garde pas son contexte. + *tn.main.n_ctx.lock().unwrap() = 4096; + let (code, _) = tn + .call("POST", "/api/tune/apply", r#"{"target":"preset"}"#) + .await; + assert_eq!(code, 202); + let st = tn.wait("application", "applying").await; + assert_eq!( + st["apply"]["error"], + json!("contexte 4096 × 1 slot(s) au lieu de 32768 × 1 : ancienne version rétablie") + ); + assert_eq!(st["apply"]["message"], json!("")); + assert_eq!( + st["apply"]["log"].as_array().unwrap().last().unwrap(), + "échec (contexte 4096 × 1 slot(s) au lieu de 32768 × 1) : retour à l'ancienne version" + ); + let preset = std::fs::read_to_string(tn.home.join("presets/dense.env")).unwrap(); + assert!( + preset.contains("UBATCH=2048"), + "preset non rétabli : {preset}" + ); + assert_eq!(tn.config()["UBATCH"], "2048", "configuration non rétablie"); + assert_eq!( + *tn.go.service_os.lock().unwrap(), + ["stop", "start", "restart", "restart"] + ); + assert!(!tn.home.join("tune.lock").exists()); + // Une sauvegarde a été faite avant d'écrire. + let backups: Vec<_> = std::fs::read_dir(tn.home.join("tune/backup")) + .unwrap() + .flatten() + .collect(); + assert_eq!(backups.len(), 1); +} + +/// Essais refusés à la composition, essais qui ne chargent pas, budget. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn essais_refuses() { + let tn = start().await; + tn.preset(&format!("{PRESET}REFUS=1024\nECHEC=512\n")); + tn.call("POST", "/api/tune", r#"{"stages":["lots"]}"#).await; + let st = tn.wait("optimisation", "running").await; + let tr = trials(&st["result"]); + assert_eq!( + tr, + [ + ( + "UBATCH=512 BATCH=2048".to_string(), + "échec".to_string(), + "mémoire insuffisante (OOM)".to_string() + ), + ( + "UBATCH=512 BATCH=1024".into(), + "échec".into(), + "mémoire insuffisante (OOM)".into() + ), + ( + "UBATCH=1024 BATCH=2048".into(), + "écarté".into(), + "refusé au lancement : [err] UBATCH=1024 refusé".into() + ), + ( + "UBATCH=2048 BATCH=4096".into(), + "mesuré".into(), + tr[3].2.clone() + ), + ], + "{tr:?}" + ); + assert!(tr[3].2.ends_with(" (seuil 3.0 %)"), "{tr:?}"); + assert_eq!(st["result"]["best"], Value::Null); + assert_eq!(st["result"]["set"], Value::Null); + // Rien à appliquer. + let (code, out) = tn + .call("POST", "/api/tune/apply", r#"{"target":"preset"}"#) + .await; + assert_eq!( + (code, out), + ( + 200, + json!({ "ok": false, "error": "rien à appliquer : la configuration actuelle reste la meilleure" }) + ) + ); + + // La référence ne se compose pas : l'optimisation échoue, le moteur est + // relancé, le verrou rendu. + tn.preset(&format!("{PRESET}REFUS=2048\n")); + tn.go.service_os.lock().unwrap().clear(); + tn.call("POST", "/api/tune", "{}").await; + let st = tn.wait("référence", "running").await; + assert_eq!( + st["error"], + json!("la configuration active ne se lance pas : [err] UBATCH=2048 refusé") + ); + assert_eq!(st["result"], Value::Null); + assert_eq!(*tn.go.service_os.lock().unwrap(), ["stop", "start"]); + // La relance échoue elle aussi : dit dans l'erreur. + *tn.go.service_os_fail.lock().unwrap() = Some(("start".into(), "boum".into())); + tn.call("POST", "/api/tune", "{}").await; + let st = tn.wait("relance", "running").await; + assert_eq!( + st["error"], + json!( + "la configuration active ne se lance pas : [err] UBATCH=2048 refusé — relance du moteur principal : boum" + ) + ); + *tn.go.service_os_fail.lock().unwrap() = None; + assert!(!tn.home.join("tune.lock").exists()); +} + +/// Annulation pendant les essais : l'essai arrêté, le résultat partiel +/// gardé. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn annulation() { + let tn = start().await; + tn.call("POST", "/api/tune", r#"{"stages":["lots"]}"#).await; + // Attendre le premier essai, puis annuler. + for _ in 0..4000 { + let st = tn.status().await; + if st["phase"] + .as_str() + .is_some_and(|p| p.starts_with("lots — ")) + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let (code, out) = tn.call("POST", "/api/tune/cancel", "").await; + assert_eq!((code, out), (200, json!({ "ok": true }))); + let st = tn.wait("annulation", "running").await; + assert_eq!(st["result"]["partial"], json!("annulé"), "{st}"); + assert!(!tn.home.join("tune.lock").exists()); + let (_, out) = tn.call("POST", "/api/tune/cancel", "").await; + assert_eq!(out, json!({ "ok": false })); + // Le stop du chat (signalé au renouvellement du bail) annule aussi. + *tn.go.lease_stop.lock().unwrap() = true; + tn.call("POST", "/api/tune", r#"{"stages":["lots"]}"#).await; + let st = tn.wait("stop", "running").await; + *tn.go.lease_stop.lock().unwrap() = false; + let annule = + st["result"]["partial"] == json!("annulé") || st["error"] == json!("optimisation annulée"); + assert!(annule, "{st}"); +} + +/// Les refus d'entrée et les routes au repos, avec les messages du Go. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn refus_d_entree() { + let tn = start().await; + assert_eq!( + tn.status().await, + json!({ "ok": true, "running": false, "applying": false }) + ); + assert_eq!( + tn.call("GET", "/api/tune", "").await, + ( + 405, + json!({ "ok": false, "error": "méthode GET refusée : POST attendu" }) + ) + ); + assert_eq!(tn.call("GET", "/api/tune/cancel", "").await.0, 405); + assert_eq!( + tn.call("POST", "/api/tune/cancel", "").await, + (200, json!({ "ok": false })) + ); + assert_eq!( + tn.call("POST", "/api/tune/apply", r#"{"target":"copy"}"#) + .await, + ( + 400, + json!({ "ok": false, "error": "aucun résultat à appliquer" }) + ) + ); + assert_eq!( + tn.call("GET", "/api/tune/last?id=dense", "").await, + (200, json!({ "ok": false })) + ); + let refus = |code: u16, e: &str| (code, json!({ "ok": false, "error": e })); + tn.go.set("config", "EXTERNAL", "1"); + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus( + 400, + "optimisation indisponible : le preset actif est une API externe" + ) + ); + tn.go.del("config", "EXTERNAL"); + tn.go.set("config", "ENGINE", "strata"); + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus( + 409, + "optimisation indisponible : le preset actif tourne sur Strata, réglé par sa propre fenêtre" + ) + ); + tn.go.del("config", "ENGINE"); + tn.go.set("config", "CTX", "4096"); + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus( + 409, + "aucun preset ne correspond à la configuration active : enregistre-la comme preset (ou bascule sur l'un d'eux) avant d'optimiser" + ) + ); + tn.go.set("config", "CTX", "32768"); + *tn.go.tune_ctx_busy.lock().unwrap() = + "2 job(s) d'arrière-plan du mode code tournent encore".into(); + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus(409, "2 job(s) d'arrière-plan du mode code tournent encore") + ); + tn.go.tune_ctx_busy.lock().unwrap().clear(); + *tn.main.busy.lock().unwrap() = true; + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus( + 409, + "le moteur traite une requête (discussion, tâche ou client /v1) : réessaie une fois libre" + ) + ); + *tn.main.busy.lock().unwrap() = false; + *tn.go.go_busy.lock().unwrap() = Some("génération en cours".into()); + assert_eq!( + tn.call("POST", "/api/tune", "{}").await, + refus(409, "génération en cours") + ); + *tn.go.go_busy.lock().unwrap() = None; + // Une optimisation tenue par un autre processus vivant (« loki tune ») : + // refus, et l'interface le voit. + let other = json!({ "owner": { "pid": 1 }, "via": "cli", "phase": "essais", "since": 1700000000, "main_was_active": false }); + std::fs::write(tn.home.join("tune.lock"), other.to_string()).unwrap(); + assert_eq!(tn.call("POST", "/api/tune", "{}").await, refus(409, BUSY)); + assert_eq!(tn.status().await["elsewhere"], json!("cli")); + std::fs::remove_file(tn.home.join("tune.lock")).unwrap(); + assert_eq!(tn.status().await.get("elsewhere"), None); + assert!(tn.go.lease_whys.lock().unwrap().iter().all(|w| w == "tune")); +} + +/// Un verrou périmé (Loki tué en plein essai) : l'essai orphelin est arrêté +/// — identité complète vérifiée — avant que l'optimisation ne reprenne la +/// place ; une application interrompue est défaite. +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn orphelin_et_application_interrompue() { + use std::os::unix::process::CommandExt; + let tn = start().await; + let mut child = std::process::Command::new("sleep") + .arg("60") + .process_group(0) + .spawn() + .unwrap(); + let pid = i64::from(child.id()); + let (start, exe) = loki_rs::tune::lock::proc_identity(pid).unwrap(); + let dead = json!({ "pid": 999999, "start": "x" }); + let write = |trial: Value, extra: Value| { + let mut o = json!({ "owner": dead, "via": "cli", "phase": "essais", "since": 1, "trial": trial, "main_was_active": false }); + for (k, v) in extra.as_object().unwrap() { + o[k] = v.clone(); + } + std::fs::write(tn.home.join("tune.lock"), o.to_string()).unwrap(); + }; + // Identité qui ne concorde pas (PID recyclé) : on ne tue rien. + write( + json!({ "pid": pid, "start": format!("{start}x"), "exes": [exe] }), + json!({}), + ); + tn.call("POST", "/api/tune", r#"{"stages":["aucune"]}"#) + .await; + tn.wait("essais", "running").await; + assert!( + child.try_wait().unwrap().is_none(), + "processus tué sur la foi du seul PID" + ); + // Identité complète : arrêté. + write( + json!({ "pid": pid, "start": start, "exes": [exe] }), + json!({}), + ); + tn.call("POST", "/api/tune", r#"{"stages":["aucune"]}"#) + .await; + tn.wait("orphelin", "running").await; + let mut gone = false; + for _ in 0..300 { + if child.try_wait().unwrap().is_some() { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(gone, "essai orphelin toujours en vie"); + + // Application interrompue : la sauvegarde revient, preset et + // configuration. + let old = std::fs::read_to_string(tn.home.join("presets/dense.env")).unwrap(); + let backup = tn.home.join("vieux.env"); + std::fs::write(&backup, &old).unwrap(); + tn.preset(&PRESET.replace("UBATCH=2048", "UBATCH=999")); + write( + Value::Null, + json!({ "phase": "application", "backup": backup, "preset_id": "dense", "preset_name": "Dense" }), + ); + tn.call("POST", "/api/tune", r#"{"stages":["aucune"]}"#) + .await; + tn.wait("application interrompue", "running").await; + assert_eq!( + std::fs::read_to_string(tn.home.join("presets/dense.env")).unwrap(), + old + ); + assert_eq!(tn.config()["UBATCH"], "2048"); +} + +/// Sans la façade pour l'optimiseur, ses routes partent au Go. +#[tokio::test] +async fn sans_la_facade() { + let tmp = tempfile::tempdir().unwrap(); + let (facade, _go, _stop) = fakego::start(tmp.path().to_path_buf()).await; + for p in [ + "/api/tune", + "/api/tune/status", + "/api/tune/cancel", + "/api/tune/apply", + "/api/tune/last", + ] { + let (c, b) = fakego::send(facade, "GET", p, &[], "").await; + assert_eq!(c.as_u16(), 200); + assert_eq!(String::from_utf8_lossy(&b), format!("go:{p}")); + } +} diff --git a/rust/loki-rs/tests/tune_core.rs b/rust/loki-rs/tests/tune_core.rs new file mode 100644 index 0000000..39642f9 --- /dev/null +++ b/rust/loki-rs/tests/tune_core.rs @@ -0,0 +1,404 @@ +//! Lot 28 : la partie pure de l'optimiseur, rejouée sur le journal tiré du Go +//! (`internal/loki/parity_tune_test.go` → `tests/parity_tune/core.json`). + +use std::collections::HashMap; + +use loki_rs::bench::BenchResult; +use loki_rs::tune::core::{self as tc, Cfg, Env, Knob, Launch, PerfRec}; +use serde_json::{Value, json}; + +fn fixture() -> Value { + serde_json::from_str(include_str!("parity_tune/core.json")).expect("journal") +} + +fn s(v: &Value) -> &str { + v.as_str().unwrap_or("") +} + +fn cfg(v: &Value) -> Cfg { + serde_json::from_value(if v.is_null() { json!({}) } else { v.clone() }).unwrap() +} + +fn strs(v: &Value) -> Vec { + serde_json::from_value(if v.is_null() { json!([]) } else { v.clone() }).unwrap() +} + +/// Une liste Go : `null` et `[]` se valent ici (tranche nil ou vide). +fn list(v: Vec) -> Value { + json!(v) +} + +fn same_list(got: Vec, want: &Value, what: &str) { + assert_eq!(list(got), list(strs(want)), "{what}"); +} + +const KNOBS: [(&str, Knob); 5] = [ + ("BATCH", tc::KNOB_BATCH), + ("UBATCH", tc::KNOB_UBATCH), + ("THREADS", tc::KNOB_THREADS), + ("FIT_TARGET", tc::KNOB_FIT_TARGET), + ("SPEC_N_MAX", tc::KNOB_SPEC_N), +]; + +#[test] +fn extra_args_jeton_par_jeton() { + let fx = fixture(); + for c in fx["extra"].as_array().unwrap() { + let input = s(&c["in"]); + let spans: Vec = tc::arg_spans(input) + .into_iter() + .map(|sp| json!([sp.start, sp.end, sp.val])) + .collect(); + let want = if c["spans"].is_null() { + json!([]) + } else { + c["spans"].clone() + }; + assert_eq!(Value::Array(spans), want, "spans {input:?}"); + same_list( + loki_rs::models::split_args(input), + &c["split"], + &format!("split {input:?}"), + ); + let ub = tc::KNOB_UBATCH.flags; + let t = tc::KNOB_THREADS.flags; + for (key, got) in [ + ("set_ub", tc::extra_set(input, ub, "2048")), + ("drop_ub", tc::extra_set(input, ub, "")), + ("set_t", tc::extra_set(input, t, "7")), + ("set_t_space", tc::extra_set(input, t, "a b")), + ("set_b", tc::extra_set(input, tc::KNOB_BATCH.flags, "4096")), + ( + "drop_ts", + tc::extra_drop(input, 1, &["-ts", "--tensor-split"]), + ), + ( + "drop_cmoe", + tc::extra_drop(input, 0, &["-cmoe", "--cpu-moe"]), + ), + ("add_bs", tc::extra_add_switch(input, "--backend-sampling")), + ] { + assert_eq!(got, s(&c[key]), "{key} {input:?}"); + } + let (dropped, removed) = tc::extra_drop_experts(input); + assert_eq!(dropped, s(&c["drop_experts"]), "drop_experts {input:?}"); + same_list(removed, &c["removed"], &format!("removed {input:?}")); + let split = loki_rs::models::split_args(input); + same_list( + tc::core_args(&split, false), + &c["core"], + &format!("core {input:?}"), + ); + same_list( + tc::core_args(&split, true), + &c["core_optin"], + &format!("core opt-in {input:?}"), + ); + } +} + +#[test] +fn reglages_par_cle_ou_par_drapeau() { + let fx = fixture(); + for c in fx["knobs"].as_array().unwrap() { + let base = cfg(&c["cfg"]); + for (name, k) in KNOBS { + assert_eq!(k.get(&base), s(&c["get"][name]), "get {name} {base:?}"); + assert_eq!( + k.set(&base, "99"), + cfg(&c["set"][name]), + "set {name} {base:?}" + ); + assert_eq!( + k.set(&base, ""), + cfg(&c["unset"][name]), + "unset {name} {base:?}" + ); + } + } +} + +fn env(v: &Value) -> Env { + let map = |x: &Value| -> HashMap { + serde_json::from_value(if x.is_null() { json!({}) } else { x.clone() }).unwrap() + }; + Env { + help: s(&v["help"]).to_string(), + gpus: v["gpus"].as_i64().unwrap(), + vram_mib: v["vram_mib"].as_i64().unwrap(), + model_bytes: v["model_bytes"].as_i64().unwrap(), + moe: v["moe"].as_bool().unwrap(), + phys_cores: v["phys_cores"].as_i64().unwrap(), + smi: v["smi"].as_bool().unwrap(), + arg_env: map(&v["arg_env"]), + user_env: map(&v["user_env"]), + queues_env: v["queues_env"].as_bool().unwrap(), + placement: v["placement"].as_bool().unwrap(), + opt_in: v["opt_in"].as_bool().unwrap(), + } +} + +#[test] +fn essais_de_chaque_etape() { + let fx = fixture(); + let mut n = 0; + for c in fx["cands"].as_array().unwrap() { + let (base, e) = (cfg(&c["cfg"]), env(&c["env"])); + let what = format!("{} / {}", s(&c["cfg_name"]), s(&c["env_name"])); + assert_eq!( + json!(tc::weights_on_cpu(&base, &e)), + c["weights_on_cpu"], + "{what} : poids sur CPU" + ); + for (name, generate) in tc::STAGES { + let (cands, why) = generate(&base, &e); + let want = &c["stages"][name]; + assert_eq!(why, s(&want["why"]), "{what} / {name} : pourquoi"); + let got: Vec = cands + .iter() + .map(|x| json!({ "stage": x.stage, "label": x.label, "cfg": x.cfg })) + .collect(); + assert_eq!(Value::Array(got), want["cands"], "{what} / {name} : essais"); + n += cands.len(); + } + } + assert!(n > 100, "{n} essais seulement"); + for c in fx["stage_wanted"].as_array().unwrap() { + let only = strs(&c["only"]); + assert_eq!( + json!(tc::stage_wanted(s(&c["name"]), &only)), + c["wanted"], + "{} {only:?}", + c["name"] + ); + } +} + +#[test] +fn garde_fous() { + let fx = fixture(); + for c in fx["denature"].as_array().unwrap() { + let name = s(&c["name"]); + let b: Launch = serde_json::from_value(c["base"].clone()).unwrap(); + let k: Launch = serde_json::from_value(c["cand"].clone()).unwrap(); + assert_eq!(tc::denature(&b, &k, false), s(&c["denature"]), "{name}"); + assert_eq!( + tc::denature(&b, &k, true), + s(&c["denature_optin"]), + "{name} opt-in" + ); + assert_eq!( + json!(tc::launch_key(&b) == tc::launch_key(&k)), + c["same_key"], + "{name} : même lancement" + ); + } + for c in fx["cfg_check"].as_array().unwrap() { + let name = s(&c["name"]); + let (b, k) = (cfg(&c["base"]), cfg(&c["cand"])); + let err = |r: Result<(), String>| r.err().unwrap_or_default(); + assert_eq!(err(tc::cfg_check(&b, &k, false)), s(&c["check"]), "{name}"); + assert_eq!( + err(tc::cfg_check(&b, &k, true)), + s(&c["check_optin"]), + "{name} opt-in" + ); + same_list(tc::diff(&b, &k), &c["diff"], &format!("{name} : diff")); + } + let content = s(&fx["patch"]["content"]); + let base = loki_rs::presets::parse_env(content); + for c in fx["patch"]["cases"].as_array().unwrap() { + let name = s(&c["name"]); + let (out, err) = match tc::patch_preset(content, &base, &cfg(&c["win"])) { + Ok(o) => (o, String::new()), + Err(e) => (String::new(), e), + }; + assert_eq!( + (out.as_str(), err.as_str()), + (s(&c["out"]), s(&c["error"])), + "{name}" + ); + } +} + +#[test] +fn journal_d_un_essai() { + let fx = fixture(); + for c in fx["logs"].as_array().unwrap() { + let log = s(&c["log"]); + let (off, total, kv) = tc::log_facts(log); + assert_eq!( + json!([off, total, kv]), + json!([c["offloaded"], c["total"], c["kv"]]), + "{log:?}" + ); + assert_eq!(tc::load_failure(log), s(&c["failure"]), "{log:?}"); + } +} + +#[test] +fn score_et_departage() { + let fx = fixture(); + for c in fx["weights"].as_array().unwrap() { + let recs: Vec = serde_json::from_value(if c["recs"].is_null() { + json!([]) + } else { + c["recs"].clone() + }) + .unwrap(); + assert_eq!( + serde_json::to_value(tc::weights_from(&recs)).unwrap(), + c["weights"], + "{}", + c["name"] + ); + } + let w = tc::Weights { + k: 2000.0, + g: 600.0, + f_cold: 0.05, + from: String::new(), + }; + for c in fx["runs"].as_array().unwrap() { + let bench: Option = serde_json::from_value(c["bench"].clone()).unwrap(); + let (run, err) = match tc::run_from(bench.as_ref(), &w) { + Ok(r) => (r, String::new()), + Err(e) => (tc::Run::default(), e), + }; + assert_eq!( + serde_json::to_value(&run).unwrap(), + c["run"], + "{}", + c["name"] + ); + assert_eq!(err, s(&c["error"]), "{}", c["name"]); + } + let f = |v: &Value| -> Vec { + serde_json::from_value(if v.is_null() { json!([]) } else { v.clone() }).unwrap() + }; + for c in fx["wins"].as_array().unwrap() { + let (b, k) = (f(&c["base"]), f(&c["cand"])); + let (gain, need, ok) = tc::wins(&b, &k); + let n = |k: &str| c[k].as_f64().unwrap(); + assert_eq!( + (gain, need, ok, tc::spread(&b), tc::mean(&k)), + ( + n("gain"), + n("need"), + c["wins"].as_bool().unwrap(), + n("spread_base"), + n("mean_cand") + ), + "{b:?} {k:?}" + ); + } + for c in fx["worth_repeat"].as_array().unwrap() { + assert_eq!( + json!(tc::worth_repeat( + &f(&c["base"]), + c["first"].as_f64().unwrap() + )), + c["worth"], + "{c}" + ); + } + for c in fx["headroom"].as_array().unwrap() { + let gpus: Vec<(i64, i64)> = serde_json::from_value(if c["gpus"].is_null() { + json!([]) + } else { + c["gpus"].clone() + }) + .unwrap(); + let visible: Vec = serde_json::from_value(if c["visible"].is_null() { + json!([]) + } else { + c["visible"].clone() + }) + .unwrap(); + let got = tc::headroom(&gpus, &visible); + assert_eq!( + json!([got.unwrap_or(0), got.is_some()]), + json!([c["free"], c["ok"]]), + "{c}" + ); + } + for c in fx["visible"].as_array().unwrap() { + let got = tc::visible_gpus(&HashMap::from([( + "CUDA_VISIBLE_DEVICES".to_string(), + s(&c["in"]).to_string(), + )])); + let want: Vec = serde_json::from_value(if c["out"].is_null() { + json!([]) + } else { + c["out"].clone() + }) + .unwrap(); + assert_eq!(got, want, "{c}"); + } + for c in fx["force_addr"].as_array().unwrap() { + assert_eq!( + list(tc::force_addr(&strs(&c["in"]), "127.0.0.1", "41234")), + c["out"], + "{c}" + ); + } + for c in fx["trial_config"].as_array().unwrap() { + let (text, err) = match tc::trial_config_text(&cfg(&c["cfg"])) { + Ok(t) => (t, String::new()), + Err(e) => (String::new(), e), + }; + assert_eq!( + (text.as_str(), err.as_str()), + (s(&c["text"]), s(&c["error"])), + "{c}" + ); + } +} + +/// Le verrou : le fichier écrit octet pour octet comme le Go, la clé de +/// prise, les binaires comparés par leur nom. +#[test] +fn verrou_au_format_du_go() { + use loki_rs::tune::lock::{Owner, held_key, same_exe}; + let fx = fixture(); + for c in fx["lock_owners"].as_array().unwrap() { + let o: Owner = serde_json::from_str(s(&c["json"])).unwrap(); + assert_eq!(serde_json::to_string(&o).unwrap(), s(&c["json"])); + assert_eq!(held_key(&o), s(&c["key"])); + } + for c in fx["same_exe"].as_array().unwrap() { + assert_eq!( + same_exe(s(&c["a"]), s(&c["b"])), + c["same"].as_bool().unwrap(), + "{c}" + ); + } +} + +/// Le résultat : enregistré et montré comme le Go l'écrit. +#[test] +fn resultat_au_format_du_go() { + use loki_rs::tune::job::TuneResult; + let fx = fixture(); + for r in fx["results"].as_array().unwrap() { + let res: TuneResult = serde_json::from_str(s(r)).unwrap(); + assert_eq!(serde_json::to_string(&res).unwrap(), s(r)); + } +} + +/// Après application : la configuration est-elle l'essai retenu ? +#[test] +fn application_conforme() { + use loki_rs::tune::job::{TuneResult, applied_mismatch}; + let fx = fixture(); + let a = &fx["applied"]; + let res = TuneResult { + base_fp: s(&a["base_fp"]).to_string(), + set: serde_json::from_value(a["set"].clone()).unwrap(), + old: serde_json::from_value(a["old"].clone()).unwrap(), + ..Default::default() + }; + for c in a["cases"].as_array().unwrap() { + assert_eq!(applied_mismatch(&cfg(&c["now"]), &res), s(&c["why"]), "{c}"); + } +}