Hitonabi
41aeb111d7
- docs/AUTONOMIE_PLAN.md: 3 Saeulen + Werkstatt, Etappen mit Verifikation, Leitplanken,
Definition-of-Done, Session-Start-Checkliste — naechste Session startet mit 'leg los'
- deploy/bench/{model-bench,brain-bench}.sh: die 02.07.-Benches als versionierte Harnesses
(Grundlage der Modell-Selbst-Evaluation E5; Quoting-/pipefail-Lehren eingebaut)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
41 lines
2.0 KiB
Bash
41 lines
2.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Brain-Config-Matrix (Referenz vom 02.07.2026): misst Qwen3.6 in mehreren Betriebsarten
|
|
# (MTP n-max / ohne Spec / KV-Quant) auf separatem Port. Für künftige Re-Benchmarks nach
|
|
# Engine-Updates oder bei neuen Hirn-Kandidaten. Ergebnis-Referenz (02.07., Vulkan, b9843):
|
|
# mtp3 78,6 t/s · mtp4 63,7 · ohne Spec 62,4 · mtp3+KVq8 78,6 (=gratis) · nospec+KVq8 62,1
|
|
set -u
|
|
MODEL="${BRAIN_GGUF:-/srv/models/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf}"
|
|
BIN="${BENCH_BIN:-/opt/llamacpp-vulkan/llama-server}"
|
|
PORT="${BENCH_PORT:-5899}"
|
|
BASE="-c 65536 -ngl 999 -fa on --no-mmap --jinja --parallel 1"
|
|
|
|
run() { # $1=name $2=extra-flags
|
|
echo "=== $1 ==="
|
|
$BIN -m "$MODEL" --host 127.0.0.1 --port "$PORT" $BASE $2 >/tmp/brain-bench-server.log 2>&1 &
|
|
local PID=$!
|
|
for i in $(seq 1 120); do
|
|
curl -s --max-time 2 "http://127.0.0.1:$PORT/health" | grep -q ok && break
|
|
sleep 2
|
|
kill -0 $PID 2>/dev/null || { echo " LOAD-CRASH"; tail -2 /tmp/brain-bench-server.log; return; }
|
|
done
|
|
for r in 0 1 2; do
|
|
curl -s --max-time 300 -X POST "http://127.0.0.1:$PORT/completion" -H "Content-Type: application/json" \
|
|
-d '{"prompt":"Erklaere in drei kurzen Saetzen, was ein Mixture-of-Experts-Modell ist.","n_predict":150,"temperature":0}' \
|
|
> /tmp/brain-bench-probe.json
|
|
[ "$r" = "0" ] && continue # Warmlauf verwerfen
|
|
python3 - <<'PY'
|
|
import json
|
|
t = json.load(open("/tmp/brain-bench-probe.json")).get("timings", {})
|
|
print(f" tg={t.get('predicted_per_second',0):.1f} t/s"
|
|
+ (f" · draft {t.get('draft_n_accepted')}/{t.get('draft_n')}" if t.get('draft_n') else ""))
|
|
PY
|
|
done
|
|
kill $PID 2>/dev/null; wait $PID 2>/dev/null; sleep 3
|
|
}
|
|
|
|
run "mtp3 (live-Config)" "--spec-type draft-mtp --spec-draft-n-max 3 -ctk q8_0 -ctv q8_0"
|
|
run "mtp4" "--spec-type draft-mtp --spec-draft-n-max 4 -ctk q8_0 -ctv q8_0"
|
|
run "ohne Spec" "-ctk q8_0 -ctv q8_0"
|
|
run "mtp3 KV-f16" "--spec-type draft-mtp --spec-draft-n-max 3"
|
|
echo "BENCH_DONE"
|