{ "applied": { "base_fp": "f0a2be841ae1baac66a6bd72e4dd3077006e78bb", "cases": [ { "now": { "MODEL": "m.gguf", "UBATCH": "2048" }, "why": "" }, { "now": { "MODEL": "m.gguf", "UBATCH": "1024" }, "why": "la configuration appliquée diffère de l'essai retenu sur UBATCH" }, { "now": { "MODEL": "autre.gguf", "UBATCH": "2048" }, "why": "la configuration a changé ailleurs que sur les clés réglées depuis la mesure" }, { "now": { "EXTRA_ARGS": "-t 4", "MODEL": "m.gguf", "UBATCH": "2048" }, "why": "la configuration appliquée diffère de l'essai retenu sur EXTRA_ARGS" }, { "now": { "MODEL": "m.gguf", "PORT": "9", "UBATCH": "2048" }, "why": "la configuration a changé ailleurs que sur les clés réglées depuis la mesure" } ], "old": { "EXTRA_ARGS": "-t 8", "UBATCH": "512" }, "set": { "EXTRA_ARGS": "", "UBATCH": "2048" } }, "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "--fit resterait inactif (--tensor-split)" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 1 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 4096 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "65536", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "2", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=2", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "4", "TEMP": "0.6", "TOP_K": "20" }, "label": "SPEC_N_MAX=4", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --backend-sampling", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "placement auto (--fit) au lieu des experts placés à la main au lieu de --tensor-split", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 7 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cfg_name": "riche", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 1024 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 2048 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 3 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=3", "stage": "threads" }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 2 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "label": "THREADS=2", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "1" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "32768", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "32768", "FIT_TARGET": "1536", "MODEL": "m.gguf" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "MODEL": "m.gguf" }, "cfg_name": "simple", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "1" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "cfg_name": "contexte non chiffré", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "3" }, "label": "THREADS=3", "stage": "threads" }, { "cfg": { "EXTRA_ARGS": "-ot exps=CPU", "MODEL": "m.gguf", "THREADS": "2" }, "label": "THREADS=2", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "1" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "32768", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (--tensor-split)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPEC": "auto", "SPLIT_MODE": "tensor" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "SPLIT_MODE=tensor" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPLIT_MODE": "tensor" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "cfg_name": "split tensor", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "32768", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "32768", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ts 1,1", "MODEL": "m.gguf", "SPLIT_MODE": "tensor" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "1", "UBATCH": "1024" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "auto", "THREADS": "7", "UBATCH": "1024" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "EXTRA_ARGS": "--backend-sampling", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "32", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=32", "stage": "délestage" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "THREADS": "7", "UBATCH": "1024" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "UBATCH": "1024" }, "label": "threads omis (cœurs physiques)", "stage": "threads" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "4", "UBATCH": "1024" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "cfg_name": "lots posés", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "on", "CUDA_LAUNCH_QUEUES": "4X", "FIT_TARGET": "1536", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "THREADS": "7", "UBATCH": "1024" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "1" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (couches --n-cpu-moe sur CPU)" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "-bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4" }, "cfg_name": "spéculation", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "draft", "SPEC_N_MAX": "4" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "3" }, "label": "THREADS=3", "stage": "threads" }, { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "On", "EXTRA_ARGS": "--n-cpu-moe 10 -bs", "MODEL": "m.gguf", "SPEC": "draft", "SPEC_N_MAX": "4", "THREADS": "2" }, "label": "THREADS=2", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "1" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [], "why": "rien à essayer" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "off", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "8192", "CUDA_LAUNCH_QUEUES": "4x", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [ { "cfg": { "CTX": "8192", "FIT_TARGET": "1024,2048", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1024,2048", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "2048,1024", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=2048,1024", "stage": "marges" }, { "cfg": { "CTX": "8192", "FIT_TARGET": "1536", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "FIT_TARGET=1536", "stage": "marges" } ], "why": "" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "8192", "EXTRA_ARGS": "--backend-sampling", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "rien n'est placé à la main" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512", "SPEC": "ngram" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "7" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "THREADS": "4" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram" }, "cfg_name": "ngram", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [], "why": "tous les poids sur GPU" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "8192", "MODEL": "m.gguf", "SPEC": "ngram", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "8192", "CUDA_GRAPH_OPT": "on", "MODEL": "m.gguf", "SPEC": "ngram" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [], "why": "tous les poids sur GPU" } }, "weights_on_cpu": false }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "riche", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans placement", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --nofit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --nofit-target MiB0,MiB1,... target margin per device for --nofit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-x MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans --fit-target", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "ce moteur ne connaît pas --fit-target" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": false, "user_env": null, "vram_mib": 28672 }, "env_name": "sans nvidia-smi", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "VRAM libre non mesurable (nvidia-smi)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": false, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "sans opt-in", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "une carte", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 3, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "trois cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 0, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "cœurs inconnus", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [], "why": "nombre de cœurs physiques inconnu" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 2, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "deux cœurs", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 1", "MODEL": "m.gguf" }, "label": "THREADS=1", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": { "LLAMA_ARG_BACKEND_SAMPLING": "1" }, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 85899345920, "moe": true, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": true, "smi": true, "user_env": { "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "64" }, "vram_mib": 28672 }, "env_name": "environnement posé", "stages": { "délestage": { "cands": [], "why": "GGML_OP_OFFLOAD_MIN_BATCH déjà posé dans l'environnement" }, "files CUDA": { "cands": [], "why": "CUDA_SCALE_LAUNCH_QUEUES déjà posé dans l'environnement" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=4096", "stage": "lots" }, { "cfg": { "BATCH": "8192", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "4096" }, "label": "UBATCH=4096 BATCH=8192", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 2, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 10737418240, "moe": false, "opt_in": true, "phys_cores": 8, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 28672 }, "env_name": "dense deux cartes", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "off", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=off", "stage": "files CUDA" }, { "cfg": { "CTX": "16384", "CUDA_LAUNCH_QUEUES": "4x", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_LAUNCH_QUEUES=4x", "stage": "files CUDA" } ], "why": "" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "--fit inactif (surcharge de tenseurs)" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "SPEC": "auto" }, "label": "SPEC=auto", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0 --backend-sampling", "MODEL": "m.gguf" }, "label": "--backend-sampling", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-t 0", "MODEL": "m.gguf" }, "label": "placement auto (--fit) au lieu des experts placés à la main", "stage": "placement" } ], "why": "" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 1, "help": "-ngl, --gpu-layers, --n-gpu-layers N max. number of layers to store in VRAM, either an exact number, 'auto', or 'all'\n-fit, --fit [on|off] whether to adjust unset arguments to fit in device memory\n-fitt, --fit-target MiB0,MiB1,... target margin per device for --fit\n--spec-type [none,draft-simple,draft-mtp,ngram-mod]\n--spec-draft-n-max N\n-bs, --backend-sampling\n--load-mode {auto,none,mmap,mlock,mmap+mlock,dio}\n", "model_bytes": 21474836480, "moe": false, "opt_in": false, "phys_cores": 8, "placement": false, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 8192 }, "env_name": "modèle plus gros que la VRAM", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [], "why": "non demandé (« inclure les options opt-in »)" }, "placement": { "cands": [], "why": "non demandé (« inclure le placement »)" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 7", "MODEL": "m.gguf" }, "label": "THREADS=7", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 4", "MODEL": "m.gguf" }, "label": "THREADS=4", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "cfg_name": "cpu-moe", "env": { "arg_env": null, "gpus": 1, "help": "", "model_bytes": 0, "moe": false, "opt_in": true, "phys_cores": 4, "placement": true, "queues_env": false, "smi": true, "user_env": null, "vram_mib": 0 }, "env_name": "moteur ancien", "stages": { "délestage": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "128" }, "label": "OP_OFFLOAD_MIN_BATCH=128", "stage": "délestage" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "OP_OFFLOAD_MIN_BATCH": "512" }, "label": "OP_OFFLOAD_MIN_BATCH=512", "stage": "délestage" } ], "why": "" }, "files CUDA": { "cands": [], "why": "une seule carte" }, "lots": { "cands": [ { "cfg": { "BATCH": "1024", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "512" }, "label": "UBATCH=512 BATCH=1024", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "1024" }, "label": "UBATCH=1024 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "2048", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=2048", "stage": "lots" }, { "cfg": { "BATCH": "4096", "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf", "UBATCH": "2048" }, "label": "UBATCH=2048 BATCH=4096", "stage": "lots" } ], "why": "" }, "marges": { "cands": [], "why": "une seule carte" }, "opt-in": { "cands": [ { "cfg": { "CTX": "16384", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-cmoe -t 0", "MODEL": "m.gguf" }, "label": "CUDA_GRAPH_OPT=on", "stage": "opt-in" } ], "why": "" }, "placement": { "cands": [], "why": "ce moteur ne connaît pas --fit" }, "threads": { "cands": [ { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 3", "MODEL": "m.gguf" }, "label": "THREADS=3", "stage": "threads" }, { "cfg": { "CTX": "16384", "EXTRA_ARGS": "-cmoe -t 2", "MODEL": "m.gguf" }, "label": "THREADS=2", "stage": "threads" } ], "why": "" } }, "weights_on_cpu": true } ], "cfg_check": [ { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "", "check_optin": "", "diff": null, "name": "identique" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20", "UBATCH": "2048" }, "check": "", "check_optin": "", "diff": [ "BATCH : (absent) → 4096", "UBATCH : (absent) → 2048" ], "name": "lots" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "32768", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "clé CTX intouchable", "check_optin": "clé CTX intouchable", "diff": [ "CTX : 65536 → 32768" ], "name": "contexte" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "auto", "TEMP": "0.6", "TOP_K": "20" }, "check": "clé SPEC intouchable", "check_optin": "", "diff": [ "SPEC : mtp → auto" ], "name": "spec" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "CUDA_GRAPH_OPT": "on", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "clé CUDA_GRAPH_OPT intouchable", "check_optin": "", "diff": [ "CUDA_GRAPH_OPT : (absent) → on" ], "name": "graphes" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TOP_K": "20" }, "check": "clé TEMP intouchable", "check_optin": "clé TEMP intouchable", "diff": [ "TEMP : 0.6 → (absent)" ], "name": "retrait" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "", "check_optin": "", "diff": [ "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 4 -ts 0.6,0.4" ], "name": "extra permis" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k f16 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "EXTRA_ARGS modifié hors liste blanche", "check_optin": "EXTRA_ARGS modifié hors liste blanche", "diff": [ "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k f16 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4" ], "name": "extra interdit" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "", "check_optin": "", "diff": [ "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4" ], "name": "extra placement" }, { "base": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "cand": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --spec-draft-n-max 4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "check": "EXTRA_ARGS modifié hors liste blanche", "check_optin": "", "diff": [ "EXTRA_ARGS : -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 → -ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4 --spec-draft-n-max 4" ], "name": "spéculation dans extra" }, { "base": { "EXTRA_ARGS": "-t 8" }, "cand": {}, "check": "", "check_optin": "", "diff": [ "EXTRA_ARGS : -t 8 → (absent)" ], "name": "extra vide" } ], "denature": [ { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "", "denature_optin": "", "name": "réglages permis", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "denature": "", "denature_optin": "", "name": "identique", "same_key": true }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "", "denature_optin": "", "name": "argument de moins", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7", "--jinja" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« --jinja » au lieu de « »)", "denature_optin": "ligne de commande différente hors réglages permis (« --jinja » au lieu de « »)", "name": "argument de plus", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "f16", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« f16 » au lieu de « q8_0 »)", "denature_optin": "ligne de commande différente hors réglages permis (« f16 » au lieu de « q8_0 »)", "name": "cache kv", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7", "-cram", "4096" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« -cram » au lieu de « »)", "denature_optin": "ligne de commande différente hors réglages permis (« -cram » au lieu de « »)", "name": "cache-ram", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "32768", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« 32768 » au lieu de « 65536 »)", "denature_optin": "ligne de commande différente hors réglages permis (« 32768 » au lieu de « 65536 »)", "name": "contexte", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "0,1", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "variable CUDA_VISIBLE_DEVICES différente (« 0,1 » au lieu de « 1,0 »)", "denature_optin": "variable CUDA_VISIBLE_DEVICES différente (« 0,1 » au lieu de « 1,0 »)", "name": "cuda", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "off", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« off » au lieu de « on »)", "denature_optin": "ligne de commande différente hors réglages permis (« off » au lieu de « on »)", "name": "fa", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_CUDA_GRAPH_OPT": "1", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "variable GGML_CUDA_GRAPH_OPT différente (« 1 » au lieu de « »)", "denature_optin": "", "name": "graphes", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7", "-md", "/d.gguf" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« -md » au lieu de « »)", "denature_optin": "", "name": "modèle ébauche", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« --cache-ram » au lieu de « -ot »)", "denature_optin": "ligne de commande différente hors réglages permis (« --cache-ram » au lieu de « -ot »)", "name": "ot", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7", "--api-key", "s3cret", "--slot-save-path", "/tmp/x", "--port=9" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« --api-key » au lieu de « »)", "denature_optin": "ligne de commande différente hors réglages permis (« --api-key » au lieu de « »)", "name": "port et clé", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "2", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« 2 » au lieu de « 1 »)", "denature_optin": "ligne de commande différente hors réglages permis (« 2 » au lieu de « 1 »)", "name": "slots", "same_key": false }, { "base": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "2048", "-ub", "512", "--host", "127.0.0.1", "--port", "41000", "--parallel", "1", "-ctk", "q8_0", "-ot", "exps=CPU,per_layer_token_embd=CPU", "--cache-ram", "8192", "--flash-attn", "on" ], "env": { "CUDA_SCALE_LAUNCH_QUEUES": "4x", "CUDA_VISIBLE_DEVICES": "1,0" } }, "cand": { "bin": "", "args": [ "/bin/llama-server", "-m", "/m.gguf", "-c", "65536", "-b", "4096", "-ub", "2048", "--host", "127.0.0.1", "--port", "41001", "--parallel", "1", "-ctk", "q8_0", "-ot", "per_layer_token_embd=CPU", "--cache-ram", "12000", "--flash-attn", "on", "-t", "7", "--spec-draft-n-max", "4" ], "env": { "CUDA_VISIBLE_DEVICES": "1,0", "GGML_OP_OFFLOAD_MIN_BATCH": "128" } }, "denature": "ligne de commande différente hors réglages permis (« --spec-draft-n-max » au lieu de « »)", "denature_optin": "", "name": "spéculation", "same_key": false } ], "extra": [ { "add_bs": "--backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "", "drop_experts": "", "drop_ts": "", "drop_ub": "", "in": "", "removed": null, "set_b": "-b 4096", "set_t": "-t 7", "set_t_space": "-t 'a b'", "set_ub": "-ub 2048", "spans": null, "split": [] }, { "add_bs": "--backend-sampling", "core": [], "core_optin": [], "drop_cmoe": " ", "drop_experts": " ", "drop_ts": " ", "drop_ub": " ", "in": " ", "removed": null, "set_b": "-b 4096", "set_t": "-t 7", "set_t_space": "-t 'a b'", "set_ub": "-ub 2048", "spans": null, "split": [] }, { "add_bs": "-b 2048 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-b 2048", "drop_experts": "-b 2048", "drop_ts": "-b 2048", "drop_ub": "-b 2048", "in": "-b 2048", "removed": null, "set_b": "-b 4096", "set_t": "-b 2048 -t 7", "set_t_space": "-b 2048 -t 'a b'", "set_ub": "-b 2048 -ub 2048", "spans": [ [ 0, 2, "-b" ], [ 3, 7, "2048" ] ], "split": [ "-b", "2048" ] }, { "add_bs": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 --backend-sampling", "core": [ "--chat-template-file", "/mes modèles/tpl.jinja" ], "core_optin": [ "--chat-template-file", "/mes modèles/tpl.jinja" ], "drop_cmoe": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", "drop_experts": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", "drop_ts": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", "drop_ub": "--chat-template-file \"/mes modèles/tpl.jinja\"", "in": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512", "removed": null, "set_b": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -b 4096", "set_t": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -t 7", "set_t_space": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 512 -t 'a b'", "set_ub": "--chat-template-file \"/mes modèles/tpl.jinja\" -ub 2048", "spans": [ [ 0, 20, "--chat-template-file" ], [ 21, 46, "/mes modèles/tpl.jinja" ], [ 47, 50, "-ub" ], [ 51, 54, "512" ] ], "split": [ "--chat-template-file", "/mes modèles/tpl.jinja", "-ub", "512" ] }, { "add_bs": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", "drop_experts": "", "drop_ts": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", "drop_ub": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", "in": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40", "removed": [ "-ot blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU", "--n-cpu-moe 40" ], "set_b": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -b 4096", "set_t": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -t 7", "set_t_space": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -t 'a b'", "set_ub": "-ot 'blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU' --n-cpu-moe 40 -ub 2048", "spans": [ [ 0, 3, "-ot" ], [ 4, 38, "blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU" ], [ 39, 50, "--n-cpu-moe" ], [ 51, 53, "40" ] ], "split": [ "-ot", "blk\\.(1[0-9])\\.ffn_.*_exps\\.=CPU", "--n-cpu-moe", "40" ] }, { "add_bs": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe --backend-sampling", "core": [ "-ot", "per_layer_token_embd=CPU", "--cache-type-k", "q8_0", "--flash-attn", "on", "--chat-template-file", "/a b/t.jinja", "-ctv", "q8_0" ], "core_optin": [ "-ot", "per_layer_token_embd=CPU", "--cache-type-k", "q8_0", "--flash-attn", "on", "--chat-template-file", "/a b/t.jinja", "-ctv", "q8_0" ], "drop_cmoe": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0", "drop_experts": "-ot per_layer_token_embd=CPU --cache-type-k q8_0 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0", "drop_ts": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ctv q8_0 -cmoe", "drop_ub": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", "in": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", "removed": [ "-ot blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU", "--n-cpu-moe 40", "-cmoe" ], "set_b": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -b 4096", "set_t": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -t 7", "set_t_space": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 512 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe -t 'a b'", "set_ub": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU\" --cache-type-k q8_0 --n-cpu-moe 40 --flash-attn on -ub 2048 --chat-template-file \"/a b/t.jinja\" -ts 0.9,0.1 -ctv q8_0 -cmoe", "spans": [ [ 0, 3, "-ot" ], [ 4, 63, "blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU" ], [ 64, 78, "--cache-type-k" ], [ 79, 83, "q8_0" ], [ 84, 95, "--n-cpu-moe" ], [ 96, 98, "40" ], [ 99, 111, "--flash-attn" ], [ 112, 114, "on" ], [ 115, 118, "-ub" ], [ 119, 122, "512" ], [ 123, 143, "--chat-template-file" ], [ 144, 158, "/a b/t.jinja" ], [ 159, 162, "-ts" ], [ 163, 170, "0.9,0.1" ], [ 171, 175, "-ctv" ], [ 176, 180, "q8_0" ], [ 181, 186, "-cmoe" ] ], "split": [ "-ot", "blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU,per_layer_token_embd=CPU", "--cache-type-k", "q8_0", "--n-cpu-moe", "40", "--flash-attn", "on", "-ub", "512", "--chat-template-file", "/a b/t.jinja", "-ts", "0.9,0.1", "-ctv", "q8_0", "-cmoe" ] }, { "add_bs": "-ub=1024 -t 8 -b 4096 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ub=1024 -t 8 -b 4096", "drop_experts": "-ub=1024 -t 8 -b 4096", "drop_ts": "-ub=1024 -t 8 -b 4096", "drop_ub": "-t 8 -b 4096", "in": "-ub=1024 -t 8 -b 4096", "removed": null, "set_b": "-ub=1024 -t 8 -b 4096", "set_t": "-ub=1024 -t 7 -b 4096", "set_t_space": "-ub=1024 -t 'a b' -b 4096", "set_ub": "-ub=2048 -t 8 -b 4096", "spans": [ [ 0, 8, "-ub=1024" ], [ 9, 11, "-t" ], [ 12, 13, "8" ], [ 14, 16, "-b" ], [ 17, 21, "4096" ] ], "split": [ "-ub=1024", "-t", "8", "-b", "4096" ] }, { "add_bs": " -ts 0.9,0.1 --flash-attn on --backend-sampling", "core": [ "--flash-attn", "on" ], "core_optin": [ "--flash-attn", "on" ], "drop_cmoe": " -ts 0.9,0.1 --flash-attn on", "drop_experts": " -ts 0.9,0.1 --flash-attn on", "drop_ts": "--flash-attn on", "drop_ub": " -ts 0.9,0.1 --flash-attn on", "in": " -ts 0.9,0.1 --flash-attn on", "removed": null, "set_b": " -ts 0.9,0.1 --flash-attn on -b 4096", "set_t": " -ts 0.9,0.1 --flash-attn on -t 7", "set_t_space": " -ts 0.9,0.1 --flash-attn on -t 'a b'", "set_ub": " -ts 0.9,0.1 --flash-attn on -ub 2048", "spans": [ [ 2, 5, "-ts" ], [ 6, 13, "0.9,0.1" ], [ 15, 27, "--flash-attn" ], [ 28, 30, "on" ] ], "split": [ "-ts", "0.9,0.1", "--flash-attn", "on" ] }, { "add_bs": "-ts=1,1 -mg 0 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ts=1,1 -mg 0", "drop_experts": "-ts=1,1 -mg 0", "drop_ts": "-mg 0", "drop_ub": "-ts=1,1 -mg 0", "in": "-ts=1,1 -mg 0", "removed": null, "set_b": "-ts=1,1 -mg 0 -b 4096", "set_t": "-ts=1,1 -mg 0 -t 7", "set_t_space": "-ts=1,1 -mg 0 -t 'a b'", "set_ub": "-ts=1,1 -mg 0 -ub 2048", "spans": [ [ 0, 7, "-ts=1,1" ], [ 8, 11, "-mg" ], [ 12, 13, "0" ] ], "split": [ "-ts=1,1", "-mg", "0" ] }, { "add_bs": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 --backend-sampling", "core": [ "-ot", "per_layer_token_embd=CPU" ], "core_optin": [ "-ot", "per_layer_token_embd=CPU" ], "drop_cmoe": "-ot exps=CPU,per_layer_token_embd=CPU -ncmoe=12", "drop_experts": "-ot per_layer_token_embd=CPU", "drop_ts": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", "drop_ub": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", "in": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12", "removed": [ "-ot exps=CPU", "-ncmoe 12", "-cmoe", "--cpu-moe" ], "set_b": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -b 4096", "set_t": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -t 7", "set_t_space": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -t 'a b'", "set_ub": "-ot exps=CPU,per_layer_token_embd=CPU -cmoe --cpu-moe -ncmoe=12 -ub 2048", "spans": [ [ 0, 3, "-ot" ], [ 4, 37, "exps=CPU,per_layer_token_embd=CPU" ], [ 38, 43, "-cmoe" ], [ 44, 53, "--cpu-moe" ], [ 54, 63, "-ncmoe=12" ] ], "split": [ "-ot", "exps=CPU,per_layer_token_embd=CPU", "-cmoe", "--cpu-moe", "-ncmoe=12" ] }, { "add_bs": "-ot=per_layer_token_embd=CPU --backend-sampling", "core": [ "-ot", "per_layer_token_embd=CPU" ], "core_optin": [ "-ot", "per_layer_token_embd=CPU" ], "drop_cmoe": "-ot=per_layer_token_embd=CPU", "drop_experts": "-ot=per_layer_token_embd=CPU", "drop_ts": "-ot=per_layer_token_embd=CPU", "drop_ub": "-ot=per_layer_token_embd=CPU", "in": "-ot=per_layer_token_embd=CPU", "removed": null, "set_b": "-ot=per_layer_token_embd=CPU -b 4096", "set_t": "-ot=per_layer_token_embd=CPU -t 7", "set_t_space": "-ot=per_layer_token_embd=CPU -t 'a b'", "set_ub": "-ot=per_layer_token_embd=CPU -ub 2048", "spans": [ [ 0, 28, "-ot=per_layer_token_embd=CPU" ] ], "split": [ "-ot=per_layer_token_embd=CPU" ] }, { "add_bs": "--override-tensor exps=CPU --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "--override-tensor exps=CPU", "drop_experts": "", "drop_ts": "--override-tensor exps=CPU", "drop_ub": "--override-tensor exps=CPU", "in": "--override-tensor exps=CPU", "removed": [ "--override-tensor exps=CPU" ], "set_b": "--override-tensor exps=CPU -b 4096", "set_t": "--override-tensor exps=CPU -t 7", "set_t_space": "--override-tensor exps=CPU -t 'a b'", "set_ub": "--override-tensor exps=CPU -ub 2048", "spans": [ [ 0, 17, "--override-tensor" ], [ 18, 26, "exps=CPU" ] ], "split": [ "--override-tensor", "exps=CPU" ] }, { "add_bs": "-ot --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ot", "drop_experts": "-ot", "drop_ts": "-ot", "drop_ub": "-ot", "in": "-ot", "removed": null, "set_b": "-ot -b 4096", "set_t": "-ot -t 7", "set_t_space": "-ot -t 'a b'", "set_ub": "-ot -ub 2048", "spans": [ [ 0, 3, "-ot" ] ], "split": [ "-ot" ] }, { "add_bs": "-t --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-t", "drop_experts": "-t", "drop_ts": "-t", "drop_ub": "-t", "in": "-t", "removed": null, "set_b": "-t -b 4096", "set_t": "-t 7", "set_t_space": "-t 'a b'", "set_ub": "-t -ub 2048", "spans": [ [ 0, 2, "-t" ] ], "split": [ "-t" ] }, { "add_bs": "-t 8 -t 6 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-t 8 -t 6", "drop_experts": "-t 8 -t 6", "drop_ts": "-t 8 -t 6", "drop_ub": "-t 8 -t 6", "in": "-t 8 -t 6", "removed": null, "set_b": "-t 8 -t 6 -b 4096", "set_t": "-t 8 -t 7", "set_t_space": "-t 8 -t 'a b'", "set_ub": "-t 8 -t 6 -ub 2048", "spans": [ [ 0, 2, "-t" ], [ 3, 4, "8" ], [ 5, 7, "-t" ], [ 8, 9, "6" ] ], "split": [ "-t", "8", "-t", "6" ] }, { "add_bs": "-ub --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ub", "drop_experts": "-ub", "drop_ts": "-ub", "drop_ub": "", "in": "-ub", "removed": null, "set_b": "-ub -b 4096", "set_t": "-ub -t 7", "set_t_space": "-ub -t 'a b'", "set_ub": "-ub 2048", "spans": [ [ 0, 3, "-ub" ] ], "split": [ "-ub" ] }, { "add_bs": "\"\" -x --backend-sampling", "core": [ "", "-x" ], "core_optin": [ "", "-x" ], "drop_cmoe": "\"\" -x", "drop_experts": "\"\" -x", "drop_ts": "\"\" -x", "drop_ub": "\"\" -x", "in": "\"\" -x", "removed": null, "set_b": "\"\" -x -b 4096", "set_t": "\"\" -x -t 7", "set_t_space": "\"\" -x -t 'a b'", "set_ub": "\"\" -x -ub 2048", "spans": [ [ 0, 2, "" ], [ 3, 5, "-x" ] ], "split": [ "", "-x" ] }, { "add_bs": "\"\"x y --backend-sampling", "core": [ "", "x", "y" ], "core_optin": [ "", "x", "y" ], "drop_cmoe": "\"\"x y", "drop_experts": "\"\"x y", "drop_ts": "\"\"x y", "drop_ub": "\"\"x y", "in": "\"\"x y", "removed": null, "set_b": "\"\"x y -b 4096", "set_t": "\"\"x y -t 7", "set_t_space": "\"\"x y -t 'a b'", "set_ub": "\"\"x y -ub 2048", "spans": [ [ 0, 2, "" ], [ 2, 3, "x" ], [ 4, 5, "y" ] ], "split": [ "", "x", "y" ] }, { "add_bs": "a\"\"b --backend-sampling", "core": [ "ab" ], "core_optin": [ "ab" ], "drop_cmoe": "a\"\"b", "drop_experts": "a\"\"b", "drop_ts": "a\"\"b", "drop_ub": "a\"\"b", "in": "a\"\"b", "removed": null, "set_b": "a\"\"b -b 4096", "set_t": "a\"\"b -t 7", "set_t_space": "a\"\"b -t 'a b'", "set_ub": "a\"\"b -ub 2048", "spans": [ [ 0, 4, "ab" ] ], "split": [ "ab" ] }, { "add_bs": "'a b'c \"d\" --backend-sampling", "core": [ "a bc", "d" ], "core_optin": [ "a bc", "d" ], "drop_cmoe": "'a b'c \"d\"", "drop_experts": "'a b'c \"d\"", "drop_ts": "'a b'c \"d\"", "drop_ub": "'a b'c \"d\"", "in": "'a b'c \"d\"", "removed": null, "set_b": "'a b'c \"d\" -b 4096", "set_t": "'a b'c \"d\" -t 7", "set_t_space": "'a b'c \"d\" -t 'a b'", "set_ub": "'a b'c \"d\" -ub 2048", "spans": [ [ 0, 6, "a bc" ], [ 7, 10, "d" ] ], "split": [ "a bc", "d" ] }, { "add_bs": "--cache-type-k q8_0\t--flash-attn on\n-t 8 --backend-sampling", "core": [ "--cache-type-k", "q8_0", "--flash-attn", "on" ], "core_optin": [ "--cache-type-k", "q8_0", "--flash-attn", "on" ], "drop_cmoe": "--cache-type-k q8_0\t--flash-attn on\n-t 8", "drop_experts": "--cache-type-k q8_0\t--flash-attn on\n-t 8", "drop_ts": "--cache-type-k q8_0\t--flash-attn on\n-t 8", "drop_ub": "--cache-type-k q8_0\t--flash-attn on\n-t 8", "in": "--cache-type-k q8_0\t--flash-attn on\n-t 8", "removed": null, "set_b": "--cache-type-k q8_0\t--flash-attn on\n-t 8 -b 4096", "set_t": "--cache-type-k q8_0\t--flash-attn on\n-t 7", "set_t_space": "--cache-type-k q8_0\t--flash-attn on\n-t 'a b'", "set_ub": "--cache-type-k q8_0\t--flash-attn on\n-t 8 -ub 2048", "spans": [ [ 0, 14, "--cache-type-k" ], [ 15, 19, "q8_0" ], [ 20, 32, "--flash-attn" ], [ 33, 35, "on" ], [ 36, 38, "-t" ], [ 39, 40, "8" ] ], "split": [ "--cache-type-k", "q8_0", "--flash-attn", "on", "-t", "8" ] }, { "add_bs": "\"non fermé -b 4 --backend-sampling", "core": [ "non fermé -b 4" ], "core_optin": [ "non fermé -b 4" ], "drop_cmoe": "\"non fermé -b 4", "drop_experts": "\"non fermé -b 4", "drop_ts": "\"non fermé -b 4", "drop_ub": "\"non fermé -b 4", "in": "\"non fermé -b 4", "removed": null, "set_b": "\"non fermé -b 4 -b 4096", "set_t": "\"non fermé -b 4 -t 7", "set_t_space": "\"non fermé -b 4 -t 'a b'", "set_ub": "\"non fermé -b 4 -ub 2048", "spans": [ [ 0, 16, "non fermé -b 4" ] ], "split": [ "non fermé -b 4" ] }, { "add_bs": "x\"y z\"w -ub=1024 --backend-sampling", "core": [ "xy zw" ], "core_optin": [ "xy zw" ], "drop_cmoe": "x\"y z\"w -ub=1024", "drop_experts": "x\"y z\"w -ub=1024", "drop_ts": "x\"y z\"w -ub=1024", "drop_ub": "x\"y z\"w", "in": "x\"y z\"w -ub=1024", "removed": null, "set_b": "x\"y z\"w -ub=1024 -b 4096", "set_t": "x\"y z\"w -ub=1024 -t 7", "set_t_space": "x\"y z\"w -ub=1024 -t 'a b'", "set_ub": "x\"y z\"w -ub=2048", "spans": [ [ 0, 7, "xy zw" ], [ 8, 16, "-ub=1024" ] ], "split": [ "xy zw", "-ub=1024" ] }, { "add_bs": "-bs --backend-sampling", "core": [ "-bs" ], "core_optin": [], "drop_cmoe": "-bs", "drop_experts": "-bs", "drop_ts": "-bs", "drop_ub": "-bs", "in": "-bs", "removed": null, "set_b": "-bs -b 4096", "set_t": "-bs -t 7", "set_t_space": "-bs -t 'a b'", "set_ub": "-bs -ub 2048", "spans": [ [ 0, 3, "-bs" ] ], "split": [ "-bs" ] }, { "add_bs": "--backend-sampling -t 4 --backend-sampling", "core": [ "--backend-sampling" ], "core_optin": [], "drop_cmoe": "--backend-sampling -t 4", "drop_experts": "--backend-sampling -t 4", "drop_ts": "--backend-sampling -t 4", "drop_ub": "--backend-sampling -t 4", "in": "--backend-sampling -t 4", "removed": null, "set_b": "--backend-sampling -t 4 -b 4096", "set_t": "--backend-sampling -t 7", "set_t_space": "--backend-sampling -t 'a b'", "set_ub": "--backend-sampling -t 4 -ub 2048", "spans": [ [ 0, 18, "--backend-sampling" ], [ 19, 21, "-t" ], [ 22, 23, "4" ] ], "split": [ "--backend-sampling", "-t", "4" ] }, { "add_bs": "-b 512\t-ub 256 --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-b 512\t-ub 256", "drop_experts": "-b 512\t-ub 256", "drop_ts": "-b 512\t-ub 256", "drop_ub": "-b 512", "in": "-b 512\t-ub 256", "removed": null, "set_b": "-b 4096\t-ub 256", "set_t": "-b 512\t-ub 256 -t 7", "set_t_space": "-b 512\t-ub 256 -t 'a b'", "set_ub": "-b 512\t-ub 2048", "spans": [ [ 0, 2, "-b" ], [ 3, 6, "512" ], [ 7, 10, "-ub" ], [ 11, 14, "256" ] ], "split": [ "-b", "512", "-ub", "256" ] }, { "add_bs": "-ncmoe --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "-ncmoe", "drop_experts": "", "drop_ts": "-ncmoe", "drop_ub": "-ncmoe", "in": "-ncmoe", "removed": [ "-ncmoe " ], "set_b": "-ncmoe -b 4096", "set_t": "-ncmoe -t 7", "set_t_space": "-ncmoe -t 'a b'", "set_ub": "-ncmoe -ub 2048", "spans": [ [ 0, 6, "-ncmoe" ] ], "split": [ "-ncmoe" ] }, { "add_bs": "--n-cpu-moe 0 --cpu-moe --backend-sampling", "core": [], "core_optin": [], "drop_cmoe": "--n-cpu-moe 0", "drop_experts": "", "drop_ts": "--n-cpu-moe 0 --cpu-moe", "drop_ub": "--n-cpu-moe 0 --cpu-moe", "in": "--n-cpu-moe 0 --cpu-moe", "removed": [ "--n-cpu-moe 0", "--cpu-moe" ], "set_b": "--n-cpu-moe 0 --cpu-moe -b 4096", "set_t": "--n-cpu-moe 0 --cpu-moe -t 7", "set_t_space": "--n-cpu-moe 0 --cpu-moe -t 'a b'", "set_ub": "--n-cpu-moe 0 --cpu-moe -ub 2048", "spans": [ [ 0, 11, "--n-cpu-moe" ], [ 12, 13, "0" ], [ 14, 23, "--cpu-moe" ] ], "split": [ "--n-cpu-moe", "0", "--cpu-moe" ] } ], "force_addr": [ { "in": [ "/b", "--host", "0.0.0.0", "--port", "8080", "-m", "x" ], "out": [ "/b", "--host", "127.0.0.1", "--port", "41234", "-m", "x" ] }, { "in": [ "/b", "--host=::", "--port=1", "--port", "2", "--host" ], "out": [ "/b", "--host=127.0.0.1", "--port=41234", "--port", "41234", "--host" ] }, { "in": [ "/b", "--hostname", "x", "-p", "3" ], "out": [ "/b", "--hostname", "x", "-p", "3" ] } ], "headroom": [ { "free": 0, "gpus": null, "ok": false, "visible": null }, { "free": 4576, "gpus": [ [ 24576, 20000 ], [ 12288, 1000 ] ], "ok": true, "visible": null }, { "free": 11288, "gpus": [ [ 24576, 20000 ], [ 12288, 1000 ] ], "ok": true, "visible": [ 1 ] }, { "free": -808, "gpus": [ [ 0, 0 ], [ 8192, 9000 ] ], "ok": true, "visible": null }, { "free": 0, "gpus": [ [ 8192, 100 ] ], "ok": false, "visible": [ 3 ] } ], "knobs": [ { "cfg": {}, "get": { "BATCH": "", "FIT_TARGET": "", "SPEC_N_MAX": "", "THREADS": "", "UBATCH": "" }, "set": { "BATCH": { "BATCH": "99" }, "FIT_TARGET": { "FIT_TARGET": "99" }, "SPEC_N_MAX": { "SPEC_N_MAX": "99" }, "THREADS": { "THREADS": "99" }, "UBATCH": { "UBATCH": "99" } }, "unset": { "BATCH": {}, "FIT_TARGET": {}, "SPEC_N_MAX": {}, "THREADS": {}, "UBATCH": {} } }, { "cfg": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "get": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "set": { "BATCH": { "BATCH": "99", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "FIT_TARGET": { "BATCH": "4096", "FIT_TARGET": "99", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "SPEC_N_MAX": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "99", "THREADS": "0", "UBATCH": "1024" }, "THREADS": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "99", "UBATCH": "1024" }, "UBATCH": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "99" } }, "unset": { "BATCH": { "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "FIT_TARGET": { "BATCH": "4096", "SPEC_N_MAX": "2", "THREADS": "0", "UBATCH": "1024" }, "SPEC_N_MAX": { "BATCH": "4096", "FIT_TARGET": "1536", "THREADS": "0", "UBATCH": "1024" }, "THREADS": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "UBATCH": "1024" }, "UBATCH": { "BATCH": "4096", "FIT_TARGET": "1536", "SPEC_N_MAX": "2", "THREADS": "0" } } }, { "cfg": { "EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024" }, "get": { "BATCH": "", "FIT_TARGET": "", "SPEC_N_MAX": "", "THREADS": "8", "UBATCH": "512" }, "set": { "BATCH": { "BATCH": "99", "EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024" }, "FIT_TARGET": { "EXTRA_ARGS": "-ub 512 -t 8", "FIT_TARGET": "99", "UBATCH": "1024" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub 512 -t 8", "SPEC_N_MAX": "99", "UBATCH": "1024" }, "THREADS": { "EXTRA_ARGS": "-ub 512 -t 99", "UBATCH": "1024" }, "UBATCH": { "EXTRA_ARGS": "-ub 99 -t 8", "UBATCH": "1024" } }, "unset": { "BATCH": { "EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024" }, "FIT_TARGET": { "EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub 512 -t 8", "UBATCH": "1024" }, "THREADS": { "EXTRA_ARGS": "-ub 512", "UBATCH": "1024" }, "UBATCH": { "EXTRA_ARGS": "-t 8", "UBATCH": "1024" } } }, { "cfg": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" }, "get": { "BATCH": "1024", "FIT_TARGET": "2048", "SPEC_N_MAX": "5", "THREADS": "6", "UBATCH": "256" }, "set": { "BATCH": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 99 -fitt 2048 --spec-draft-n-max 5" }, "FIT_TARGET": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 99 --spec-draft-n-max 5" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 99" }, "THREADS": { "EXTRA_ARGS": "-ub=256 --threads 99 -b 1024 -fitt 2048 --spec-draft-n-max 5" }, "UBATCH": { "EXTRA_ARGS": "-ub=99 --threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" } }, "unset": { "BATCH": { "EXTRA_ARGS": "-ub=256 --threads 6 -fitt 2048 --spec-draft-n-max 5" }, "FIT_TARGET": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 --spec-draft-n-max 5" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub=256 --threads 6 -b 1024 -fitt 2048" }, "THREADS": { "EXTRA_ARGS": "-ub=256 -b 1024 -fitt 2048 --spec-draft-n-max 5" }, "UBATCH": { "EXTRA_ARGS": "--threads 6 -b 1024 -fitt 2048 --spec-draft-n-max 5" } } }, { "cfg": { "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "get": { "BATCH": "", "FIT_TARGET": "", "SPEC_N_MAX": "", "THREADS": "", "UBATCH": "512" }, "set": { "BATCH": { "BATCH": "99", "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "FIT_TARGET": { "EXTRA_ARGS": "-ub 512", "FIT_TARGET": "99", "UBATCH": " 128 " }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub 512", "SPEC_N_MAX": "99", "UBATCH": " 128 " }, "THREADS": { "EXTRA_ARGS": "-ub 512", "THREADS": "99", "UBATCH": " 128 " }, "UBATCH": { "EXTRA_ARGS": "-ub 99", "UBATCH": " 128 " } }, "unset": { "BATCH": { "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "FIT_TARGET": { "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "SPEC_N_MAX": { "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "THREADS": { "EXTRA_ARGS": "-ub 512", "UBATCH": " 128 " }, "UBATCH": { "UBATCH": " 128 " } } }, { "cfg": { "EXTRA_ARGS": "-t" }, "get": { "BATCH": "", "FIT_TARGET": "", "SPEC_N_MAX": "", "THREADS": "", "UBATCH": "" }, "set": { "BATCH": { "BATCH": "99", "EXTRA_ARGS": "-t" }, "FIT_TARGET": { "EXTRA_ARGS": "-t", "FIT_TARGET": "99" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-t", "SPEC_N_MAX": "99" }, "THREADS": { "EXTRA_ARGS": "-t 99" }, "UBATCH": { "EXTRA_ARGS": "-t", "UBATCH": "99" } }, "unset": { "BATCH": { "EXTRA_ARGS": "-t" }, "FIT_TARGET": { "EXTRA_ARGS": "-t" }, "SPEC_N_MAX": { "EXTRA_ARGS": "-t" }, "THREADS": {}, "UBATCH": { "EXTRA_ARGS": "-t" } } }, { "cfg": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "get": { "BATCH": "", "FIT_TARGET": "", "SPEC_N_MAX": "", "THREADS": "8", "UBATCH": "512" }, "set": { "BATCH": { "BATCH": "99", "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "FIT_TARGET": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "FIT_TARGET": "99", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "SPEC_N_MAX": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "SPEC_N_MAX": "99", "TEMP": "0.6", "TOP_K": "20" }, "THREADS": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 99 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "UBATCH": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 99 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" } }, "unset": { "BATCH": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "FIT_TARGET": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "SPEC_N_MAX": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "THREADS": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -ub 512 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" }, "UBATCH": { "CTX": "65536", "EXTRA_ARGS": "-ot \"blk\\.([0-9]+)\\.ffn_.*_exps\\.=CPU\" --n-cpu-moe 40 --cache-type-k q8_0 --flash-attn on -t 8 -ts 0.6,0.4", "KV_TYPE": "q8_0", "MMPROJ": "mmproj.gguf", "MODEL": "big.gguf", "PARALLEL": "1", "REASONING": "on", "REASONING_BUDGET": "-1", "SPEC": "mtp", "TEMP": "0.6", "TOP_K": "20" } } } ], "lock_owners": [ { "json": "{\"owner\":{\"pid\":42},\"via\":\"web\",\"phase\":\"essais\",\"since\":1700000000,\"main_was_active\":false}", "key": "web|1700000000|0" }, { "json": "{\"owner\":{\"pid\":42,\"start\":\"b0c1/123\",\"exes\":[\"/usr/bin/loki-rs\"]},\"via\":\"apply\",\"phase\":\"application\",\"since\":1700000000,\"nonce\":1700000000123456789,\"trial\":{\"pid\":43,\"start\":\"b0c1/456\",\"exes\":[\"/usr/bin/loki\",\"/opt/llama-server\"]},\"main_was_active\":true,\"backup\":\"/home/l/tune/backup/Dense-1.env\",\"preset_id\":\"Dense\",\"preset_name\":\"Dense\"}", "key": "apply|1700000000|1700000000123456789" } ], "logs": [ { "failure": "moteur arrêté : load_tensors: offloaded 49/49 layers to GPU", "kv": "q8_0/q8_0", "log": "load_tensors: offloaded 40/49 layers to GPU\nllama_kv_cache: size = 1024.00 MiB ( 8192 cells, 32 layers, 1/1 seqs), K (q8_0): 512.00 MiB, V (q8_0): 512.00 MiB\nload_tensors: offloaded 49/49 layers to GPU\n", "offloaded": 49, "total": 49 }, { "failure": "moteur arrêté : rien", "kv": "", "log": "rien", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté au chargement", "kv": "", "log": "", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté : K (f16): x V (f16): y", "kv": "f16/f16,q4_0/q8_0", "log": "K (F16): 1 MiB, V (f16): 1 MiB\nK (q4_0): 1, V (q8_0): 2\nK (f16): x V (f16): y", "offloaded": -1, "total": -1 }, { "failure": "mémoire insuffisante (OOM)", "kv": "", "log": "ggml_backend_cuda_buffer_type_alloc_buffer: cudaMalloc failed: out of memory", "offloaded": -1, "total": -1 }, { "failure": "--fit n'a pas su placer le modèle", "kv": "", "log": "common_fit: failed to fit model\n", "offloaded": -1, "total": -1 }, { "failure": "échec du chargement du modèle", "kv": "", "log": "llama_model_load: error loading model: bad magic\n", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté : derniere ligne de journal", "kv": "", "log": "ligne 1\nderniere ligne de journal \n\n", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté au chargement", "kv": "", "log": "\n \n", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté : éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "kv": "", "log": "x\néééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "offloaded": -1, "total": -1 }, { "failure": "moteur arrêté : aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\ufffd", "kv": "", "log": "x\naaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaébbbbbbbbbbbbbbbbbbbb", "offloaded": -1, "total": -1 } ], "patch": { "content": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\n", "cases": [ { "error": "", "name": "threads retirés, lots", "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=2048\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on\"\nBATCH=4096\n", "win": { "BATCH": "4096", "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on", "MODEL": "big.gguf", "UBATCH": "2048" } }, { "error": "clé CTX intouchable", "name": "contexte", "out": "", "win": { "CTX": "32768", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", "MODEL": "big.gguf", "UBATCH": "512" } }, { "error": "", "name": "rien", "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\n", "win": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", "MODEL": "big.gguf", "UBATCH": "512" } }, { "error": "", "name": "opt-in", "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\nEXTRA_ARGS=\"--cache-type-k q8_0 --flash-attn on -t 8\"\nSPEC=auto\n", "win": { "CTX": "65536", "EXTRA_ARGS": "--cache-type-k q8_0 --flash-attn on -t 8", "MODEL": "big.gguf", "SPEC": "auto", "UBATCH": "512" } }, { "error": "", "name": "retrait d'EXTRA_ARGS", "out": "# NAME=Gros MoE\n# commentaire\nMODEL=big.gguf\nCTX=65536\nUBATCH=512\n", "win": { "CTX": "65536", "MODEL": "big.gguf", "UBATCH": "512" } } ] }, "results": [ "{\"preset_id\":\"\",\"preset_name\":\"\",\"base_fp\":\"\",\"preset_fp\":\"\",\"engine\":\"\",\"engine_build\":0,\"weights\":{\"k\":0,\"g\":0,\"f_cold\":0,\"from\":\"\"},\"depth\":0,\"n_ctx\":0,\"slots\":0,\"baseline\":{\"stage\":\"\",\"label\":\"\",\"status\":\"\",\"gain\":0,\"headroom_mib\":0,\"sec\":0},\"trials\":null,\"gain\":0,\"placement\":false,\"opt_in\":false,\"at\":0}", "{\"preset_id\":\"Dense\",\"preset_name\":\"Dense\",\"base_fp\":\"abc\",\"preset_fp\":\"def\",\"engine\":\"b7000-x\",\"engine_build\":7000,\"weights\":{\"k\":2000,\"g\":600,\"f_cold\":0.05,\"from\":\"défaut\"},\"depth\":8192,\"n_ctx\":16384,\"slots\":1,\"baseline\":{\"stage\":\"référence\",\"label\":\"configuration actuelle\",\"status\":\"mesuré\",\"runs\":[{\"turn_sec\":3.25,\"cold_pp\":812.5,\"cached_pp\":1500,\"tg\":42.75,\"prose_pp\":2000,\"prose_tg\":45.5,\"depth\":8192},{\"turn_sec\":3.5,\"prose_pp\":1900,\"prose_tg\":44}],\"turn_sec\":3.375,\"gain\":0,\"headroom_mib\":-1,\"offloaded\":\"33/33\",\"sec\":61.5},\"trials\":[{\"stage\":\"lots\",\"label\":\"UBATCH=512 BATCH=1024\",\"status\":\"retenu\",\"why\":\"−7.3 % par tour (seuil 3.0 %)\",\"runs\":[{\"turn_sec\":3.25,\"cold_pp\":812.5,\"cached_pp\":1500,\"tg\":42.75,\"prose_pp\":2000,\"prose_tg\":45.5,\"depth\":8192}],\"turn_sec\":3.125,\"gain\":0.0734,\"headroom_mib\":512,\"changes\":[\"UBATCH 2048 → 512\"],\"sec\":30},{\"stage\":\"lots\",\"label\":\"UBATCH=2048 BATCH=4096\",\"status\":\"écarté\",\"why\":\"dénature : x\",\"gain\":0,\"headroom_mib\":-1,\"sec\":0}],\"best\":\"UBATCH=512 BATCH=1024\",\"gain\":0.0734,\"set\":{\"BATCH\":\"1024\",\"UBATCH\":\"512\"},\"old\":{\"BATCH\":\"\",\"UBATCH\":\"2048\"},\"changes\":[\"UBATCH 2048 → 512\",\"BATCH + 1024\"],\"placement\":true,\"partial\":\"annulé\",\"skipped\":[\"marges : une seule carte\"],\"notes\":[\"n\"],\"opt_in\":true,\"at\":1700000000}" ], "runs": [ { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "", "name": "complet", "run": { "turn_sec": 18.997317738791423, "cold_pp": 800, "cached_pp": 1026, "tg": 40, "prose_pp": 1000, "prose_tg": 50, "depth": 32768 } }, { "bench": null, "error": "mesure vide", "name": "vide", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0, "elapsed_sec": 0 }, "error": "mesure vide", "name": "sans decode", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0 }, "error": "", "name": "rapide", "run": { "turn_sec": 14, "prose_pp": 1000, "prose_tg": 50 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 0, "skipped": "petit", "reuse": 0 } }, "error": "", "name": "profondeur sautée", "run": { "turn_sec": 14, "prose_pp": 1000, "prose_tg": 50 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0, "partial": "contexte plein" } }, "error": "mesure partielle : contexte plein", "name": "partiel", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "mesure en profondeur incomplète", "name": "sans froid", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "mesure en profondeur incomplète", "name": "froid nul", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "mesure en profondeur incomplète", "name": "sans tours", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "reuse": 0 } }, "error": "mesure en profondeur incomplète", "name": "sans decode à D", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": -1, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "", "name": "cache inconnu", "run": { "turn_sec": 19.050002002002003, "cold_pp": 800, "cached_pp": 999, "tg": 40, "prose_pp": 1000, "prose_tg": 50, "depth": 32768 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 2100, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 2100, "cached": -1, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "decode_per_second": 40, "reuse": 0 } }, "error": "prefill des tours illisible", "name": "cache inconnu, rien d'autre", "run": { "turn_sec": 0, "prose_pp": 0, "prose_tg": 0 } }, { "bench": { "prompt_n": 0, "prompt_ms": 0, "prompt_per_second": 1000, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 50, "elapsed_sec": 0, "depth": { "ctx": 0, "target": 32768, "cold": { "new": 0, "cached": 0, "expected": 0, "prompt_ms": 0, "prompt_per_second": 800, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, "turns": [ { "new": 48, "cached": 30000, "expected": 30048, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 }, { "new": 48, "cached": 32300, "expected": 32348, "prompt_ms": 2000, "prompt_per_second": 0, "predicted_n": 0, "predicted_ms": 0, "predicted_per_second": 0 } ], "cached_per_second": 999, "decode_per_second": 40, "reuse": 0 } }, "error": "", "name": "rien de neuf", "run": { "turn_sec": 19.050002002002003, "cold_pp": 800, "cached_pp": 999, "tg": 40, "prose_pp": 1000, "prose_tg": 50, "depth": 32768 } } ], "same_exe": [ { "a": "/usr/local/bin/loki", "b": "/usr/local/bin/loki (deleted)", "same": true }, { "a": "/usr/local/bin/loki", "b": "/usr/bin/llama-server", "same": false }, { "a": "C:\\Loki\\loki.exe", "b": "LOKI", "same": true }, { "a": "", "b": "loki", "same": false }, { "a": "/a/llama-server", "b": "llama-server", "same": true }, { "a": "/a/b/", "b": "b", "same": true } ], "stage_wanted": [ { "name": "placement", "only": null, "wanted": true }, { "name": "placement", "only": [ "lots" ], "wanted": true }, { "name": "placement", "only": [ "Délestage" ], "wanted": true }, { "name": "placement", "only": [ "files" ], "wanted": true }, { "name": "placement", "only": [ "FILES CUDA" ], "wanted": true }, { "name": "placement", "only": [ "marges-x" ], "wanted": true }, { "name": "placement", "only": [ " threads " ], "wanted": true }, { "name": "placement", "only": [ "opt" ], "wanted": true }, { "name": "placement", "only": [ "x" ], "wanted": true }, { "name": "marges", "only": null, "wanted": true }, { "name": "marges", "only": [ "lots" ], "wanted": false }, { "name": "marges", "only": [ "Délestage" ], "wanted": false }, { "name": "marges", "only": [ "files" ], "wanted": false }, { "name": "marges", "only": [ "FILES CUDA" ], "wanted": false }, { "name": "marges", "only": [ "marges-x" ], "wanted": true }, { "name": "marges", "only": [ " threads " ], "wanted": false }, { "name": "marges", "only": [ "opt" ], "wanted": false }, { "name": "marges", "only": [ "x" ], "wanted": false }, { "name": "lots", "only": null, "wanted": true }, { "name": "lots", "only": [ "lots" ], "wanted": true }, { "name": "lots", "only": [ "Délestage" ], "wanted": false }, { "name": "lots", "only": [ "files" ], "wanted": false }, { "name": "lots", "only": [ "FILES CUDA" ], "wanted": false }, { "name": "lots", "only": [ "marges-x" ], "wanted": false }, { "name": "lots", "only": [ " threads " ], "wanted": false }, { "name": "lots", "only": [ "opt" ], "wanted": false }, { "name": "lots", "only": [ "x" ], "wanted": false }, { "name": "délestage", "only": null, "wanted": true }, { "name": "délestage", "only": [ "lots" ], "wanted": false }, { "name": "délestage", "only": [ "Délestage" ], "wanted": true }, { "name": "délestage", "only": [ "files" ], "wanted": false }, { "name": "délestage", "only": [ "FILES CUDA" ], "wanted": false }, { "name": "délestage", "only": [ "marges-x" ], "wanted": false }, { "name": "délestage", "only": [ " threads " ], "wanted": false }, { "name": "délestage", "only": [ "opt" ], "wanted": false }, { "name": "délestage", "only": [ "x" ], "wanted": false }, { "name": "threads", "only": null, "wanted": true }, { "name": "threads", "only": [ "lots" ], "wanted": false }, { "name": "threads", "only": [ "Délestage" ], "wanted": false }, { "name": "threads", "only": [ "files" ], "wanted": false }, { "name": "threads", "only": [ "FILES CUDA" ], "wanted": false }, { "name": "threads", "only": [ "marges-x" ], "wanted": false }, { "name": "threads", "only": [ " threads " ], "wanted": true }, { "name": "threads", "only": [ "opt" ], "wanted": false }, { "name": "threads", "only": [ "x" ], "wanted": false }, { "name": "files CUDA", "only": null, "wanted": true }, { "name": "files CUDA", "only": [ "lots" ], "wanted": false }, { "name": "files CUDA", "only": [ "Délestage" ], "wanted": false }, { "name": "files CUDA", "only": [ "files" ], "wanted": true }, { "name": "files CUDA", "only": [ "FILES CUDA" ], "wanted": true }, { "name": "files CUDA", "only": [ "marges-x" ], "wanted": false }, { "name": "files CUDA", "only": [ " threads " ], "wanted": false }, { "name": "files CUDA", "only": [ "opt" ], "wanted": false }, { "name": "files CUDA", "only": [ "x" ], "wanted": false }, { "name": "opt-in", "only": null, "wanted": true }, { "name": "opt-in", "only": [ "lots" ], "wanted": true }, { "name": "opt-in", "only": [ "Délestage" ], "wanted": true }, { "name": "opt-in", "only": [ "files" ], "wanted": true }, { "name": "opt-in", "only": [ "FILES CUDA" ], "wanted": true }, { "name": "opt-in", "only": [ "marges-x" ], "wanted": true }, { "name": "opt-in", "only": [ " threads " ], "wanted": true }, { "name": "opt-in", "only": [ "opt" ], "wanted": true }, { "name": "opt-in", "only": [ "x" ], "wanted": true } ], "trial_config": [ { "cfg": { "EXTRA_ARGS": "-ot \"a b\" -t 8", "MODEL": "m.gguf", "PORT": "41000" }, "error": "", "text": "EXTRA_ARGS=-ot \"a b\" -t 8\nMODEL=m.gguf\nPORT=41000\n" }, { "cfg": { "MODEL": "x\ny" }, "error": "valeur de MODEL non représentable dans un fichier de configuration", "text": "" }, { "cfg": { "A": " b ", "NAME": "é ü" }, "error": "", "text": "A=\" b \"\nNAME=\"é ü\"\n" }, { "cfg": { "A": "\"q\"", "B": "c'd", "C": "#h" }, "error": "valeur de A non représentable dans un fichier de configuration", "text": "" } ], "visible": [ { "in": "", "out": null }, { "in": "0", "out": [ 0 ] }, { "in": "1,0", "out": [ 1, 0 ] }, { "in": " 2 , 3 ", "out": [ 2, 3 ] }, { "in": "GPU-abc", "out": null }, { "in": "0,,1", "out": null }, { "in": "+1,-2", "out": [ 1, -2 ] } ], "weights": [ { "name": "vide", "recs": null, "weights": { "k": 2000, "g": 600, "f_cold": 0.05, "from": "défaut" } }, { "name": "trop peu", "recs": [ { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1000, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1001, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1002, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1003, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1004, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1005, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1006, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1007, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1008, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1009, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1010, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1011, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1012, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1013, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1014, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1015, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1016, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1017, "gen": 300 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "new": 1018, "gen": 300 } ], "weights": { "k": 2000, "g": 600, "f_cold": 0.05, "from": "défaut" } }, { "name": "télémétrie", "recs": [ { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4000, "cached": 100, "new": 500, "gen": 100 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4100, "cached": 3900, "new": 537, "gen": 113 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4200, "cached": 4000, "new": 574, "gen": 126 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4300, "cached": 100, "new": 611, "gen": 139 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4400, "cached": 4200, "new": 648, "gen": 152 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4500, "cached": 4300, "new": 685, "gen": 165 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4600, "cached": 100, "new": 722, "gen": 178 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4700, "cached": 4500, "new": 759, "gen": 191 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4800, "cached": 4600, "new": 796, "gen": 204 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 4900, "cached": 100, "new": 833, "gen": 217 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5000, "cached": 4800, "new": 870, "gen": 230 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5100, "cached": 4900, "new": 907, "gen": 243 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5200, "cached": 100, "new": 944, "gen": 256 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5300, "cached": 5100, "new": 981, "gen": 269 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5400, "cached": 5200, "new": 1018, "gen": 282 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5500, "cached": 100, "new": 1055, "gen": 295 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5600, "cached": 5400, "new": 1092, "gen": 308 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5700, "cached": 5500, "new": 1129, "gen": 321 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5800, "cached": 100, "new": 1166, "gen": 334 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 5900, "cached": 5700, "new": 1203, "gen": 347 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6000, "cached": 5800, "new": 1240, "gen": 360 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6100, "cached": 100, "new": 1277, "gen": 373 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6200, "cached": 6000, "new": 1314, "gen": 386 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6300, "cached": 6100, "new": 1351, "gen": 399 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6400, "cached": 100, "new": 1388, "gen": 412 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6500, "cached": 6300, "new": 1425, "gen": 425 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6600, "cached": 6400, "new": 1462, "gen": 438 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6700, "cached": 100, "new": 1499, "gen": 451 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6800, "cached": 6600, "new": 1536, "gen": 464 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 6900, "cached": 6700, "new": 1573, "gen": 477 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "bench", "iter": 0, "complete": true, "new": 99999, "gen": 99999 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": false, "new": 99999, "gen": 99999 } ], "weights": { "k": 1036.5, "g": 288.5, "f_cold": 0.3333333333333333, "from": "télémétrie" } }, { "name": "bornes", "recs": [ { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "task", "iter": 0, "complete": true, "total": 100, "cached": 0, "new": 50, "gen": 9000 } ], "weights": { "k": 128, "g": 8192, "f_cold": 0.05, "from": "télémétrie" } }, { "name": "pair, froid connu", "recs": [ { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1000, "gen": 200 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1010, "gen": 201 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1020, "gen": 202 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1030, "gen": 203 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1040, "gen": 204 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1050, "gen": 205 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1060, "gen": 206 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1070, "gen": 207 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1080, "gen": 208 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1090, "gen": 209 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1100, "gen": 210 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1110, "gen": 211 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1120, "gen": 212 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1130, "gen": 213 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1140, "gen": 214 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1150, "gen": 215 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1160, "gen": 216 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1170, "gen": 217 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1180, "gen": 218 }, { "at": "0001-01-01T00:00:00Z", "seq": 0, "kind": "main", "iter": 0, "complete": true, "total": 2048, "cached": 204, "new": 1190, "gen": 219 } ], "weights": { "k": 1095, "g": 209.5, "f_cold": 1, "from": "télémétrie" } } ], "wins": [ { "base": [ 20, 20.4 ], "cand": [ 19.7, 19.8 ], "gain": 0.02227722772277224, "mean_cand": 19.75, "need": 0.03, "spread_base": 0.019801980198019733, "wins": false }, { "base": [ 20, 20.4 ], "cand": [ 18 ], "gain": 0.10891089108910888, "mean_cand": 18, "need": 0.03, "spread_base": 0.019801980198019733, "wins": false }, { "base": [ 20, 20.4 ], "cand": [ 18, 18.1 ], "gain": 0.10643564356435636, "mean_cand": 18.05, "need": 0.03, "spread_base": 0.019801980198019733, "wins": true }, { "base": [ 18, 22 ], "cand": [ 18.5, 18.6 ], "gain": 0.07249999999999997, "mean_cand": 18.55, "need": 0.2, "spread_base": 0.2, "wins": false }, { "base": [], "cand": [ 1, 2 ], "gain": 0, "mean_cand": 1.5, "need": 0.03, "spread_base": 0, "wins": false }, { "base": [ 10 ], "cand": [], "gain": 0, "mean_cand": 0, "need": 0.03, "spread_base": 0, "wins": false }, { "base": [ 10, 10 ], "cand": [ 9.6, 9.6 ], "gain": 0.040000000000000036, "mean_cand": 9.6, "need": 0.03, "spread_base": 0, "wins": true }, { "base": [ 10, 10 ], "cand": [ 9.7, 9.7 ], "gain": 0.030000000000000072, "mean_cand": 9.7, "need": 0.03, "spread_base": 0, "wins": true }, { "base": [ 0 ], "cand": [ 1, 1 ], "gain": 0, "mean_cand": 1, "need": 0.03, "spread_base": 0, "wins": false }, { "base": [ 10, 12 ], "cand": [ 8, 8.2, 8.1 ], "gain": 0.26363636363636367, "mean_cand": 8.1, "need": 0.18181818181818182, "spread_base": 0.18181818181818182, "wins": true } ], "worth_repeat": [ { "base": [ 20, 20.4 ], "first": 19, "worth": true }, { "base": [ 20, 20.4 ], "first": 19.8, "worth": false }, { "base": [ 20 ], "first": 19.39, "worth": true }, { "base": [ 20 ], "first": 19.41, "worth": false }, { "base": null, "first": 1, "worth": false } ] }