Feat: Vulkan/RADV-Engine + vocab-gepruefte Spec-Drafts + Provisioning/Sync
Engine-Cutover ROCm/HIP -> Vulkan/RADV (gfx1151): +12-22% tg auf MoE (llama-bench
verifiziert, fast 53->65 t/s). ROCm-Build bleibt als Rollback unter /opt/llamacpp.
- Backend: vocab-aware Speculative Decoding. services/gguf_meta.py liest den
Tokenizer-Fingerprint (model/pre/n_vocab) direkt aus dem GGUF-Header (ohne Modell-Load);
register_model + migrate_config haengen nur VOCAB-KOMPATIBLE Drafts an (inkl. --spec-type,
das in dieser llama.cpp-Generation noetig ist). Neue Endpoints /api/models/drafts + /{id}/draft.
- Frontend: idiotensichere Spec-Draft-UI (SpecDraftModal) - nur kompatible Drafts waehlbar,
inkompatible gesperrt mit Begruendung; SPEC/SPEC?-Badge nach echtem Aktiv-Status; Rolle in AddModel.
- maintenance.py: Engine-Update-Quelle -> ggml-org/llama.cpp (Build-Nummer-Vergleich),
ENGINE_PATH=/opt/llamacpp-vulkan.
- Startup-Warmup der brains (deploy/warmup.sh, self-detaching ExecStartPost) + deploy/provision-engine.sh.
- Cleanup: tote LiteLLM gateway/config.yaml + alle Referenzen (config.py/backup.py/backup.sh) entfernt;
README + docs/memory aktualisiert.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,10 @@ import re
|
||||
import httpx
|
||||
from ruamel.yaml.scalarstring import LiteralScalarString
|
||||
|
||||
from config import CMD_TEMPLATE, CONFIG_PATH, DEFAULT_TTL, LLAMA_SWAP_URL, SPEC_DRAFT_MODEL_PATH
|
||||
from config import (
|
||||
CMD_TEMPLATE, CONFIG_PATH, DEFAULT_TTL, DRAFTS_DIR, LLAMA_SWAP_URL,
|
||||
SPEC_DRAFT_MODEL_PATH, SPEC_TYPE,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -66,6 +69,11 @@ def _parse_model(name: str, spec: dict) -> dict:
|
||||
m_draft = re.search(r"--spec-draft-model\s+([^\s]+)", cmd)
|
||||
if m_draft:
|
||||
spec_draft = os.path.basename(m_draft.group(1).replace("'", "").replace('"', ""))
|
||||
spec_type = None
|
||||
if (m_st := re.search(r"--spec-type\s+([^\s]+)", cmd)):
|
||||
spec_type = m_st.group(1)
|
||||
# Spec ist nur AKTIV, wenn BEIDES gesetzt ist (--spec-draft-model UND --spec-type).
|
||||
spec_active = bool(spec_draft and spec_type)
|
||||
parallel_match = re.search(r"--parallel\s+(\d+)", cmd)
|
||||
parallel_slots = int(parallel_match.group(1)) if parallel_match else 1
|
||||
|
||||
@@ -85,6 +93,8 @@ def _parse_model(name: str, spec: dict) -> dict:
|
||||
"incomplete": not path,
|
||||
"prompt_cache": prompt_cache,
|
||||
"spec_draft_model": spec_draft,
|
||||
"spec_type": spec_type,
|
||||
"spec_active": spec_active,
|
||||
"parallel_slots": parallel_slots,
|
||||
"capabilities": capabilities(
|
||||
name=filename or name, cmd=cmd,
|
||||
@@ -191,8 +201,10 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
|
||||
if role_lower in ("fast", "coder"):
|
||||
if "--parallel" not in cmd:
|
||||
cmd += " --parallel 2"
|
||||
if os.path.exists(SPEC_DRAFT_MODEL_PATH) and "--spec-draft-model" not in cmd:
|
||||
cmd += f" --spec-draft-model {SPEC_DRAFT_MODEL_PATH}"
|
||||
# Nur einen VOCAB-KOMPATIBLEN Draft anhängen (sonst scheitert llama.cpp).
|
||||
# Bei frischem Install existiert die GGUF noch nicht → kein Draft (später im UI setzbar).
|
||||
if "--spec-draft-model" not in cmd:
|
||||
cmd += spec_draft_flags(model_path)
|
||||
|
||||
cfg.setdefault("models", {})[model_id] = {
|
||||
"cmd": LiteralScalarString(cmd + "\n"),
|
||||
@@ -203,6 +215,101 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
|
||||
return model_id
|
||||
|
||||
|
||||
# --- Speculative-Draft / Vocab-Kompatibilität --------------------------------
|
||||
def list_drafts() -> list[dict]:
|
||||
"""Alle Draft-GGUFs in DRAFTS_DIR mit Tokenizer-Fingerprint."""
|
||||
from services import gguf_meta
|
||||
out = []
|
||||
if DRAFTS_DIR.is_dir():
|
||||
for p in sorted(DRAFTS_DIR.glob("*.gguf")):
|
||||
out.append({
|
||||
"path": str(p), "filename": p.name,
|
||||
"size_bytes": p.stat().st_size if p.exists() else None,
|
||||
"vocab": gguf_meta.fingerprint(str(p)),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def find_compatible_draft(target_path: str) -> str | None:
|
||||
"""Pfad eines vocab-kompatiblen Drafts für target_path, oder None.
|
||||
Bevorzugt MC_SPEC_DRAFT_MODEL (falls gesetzt+kompatibel), sonst der erste
|
||||
kompatible Draft in DRAFTS_DIR. None auch, wenn target (noch) fehlt (dann
|
||||
nicht verifizierbar → bewusst KEIN Draft anhängen)."""
|
||||
if not target_path or not os.path.exists(target_path):
|
||||
return None
|
||||
from services import gguf_meta
|
||||
candidates: list[str] = []
|
||||
if SPEC_DRAFT_MODEL_PATH and os.path.exists(SPEC_DRAFT_MODEL_PATH):
|
||||
candidates.append(SPEC_DRAFT_MODEL_PATH)
|
||||
for d in list_drafts():
|
||||
if d["path"] not in candidates:
|
||||
candidates.append(d["path"])
|
||||
for c in candidates:
|
||||
if gguf_meta.compatible(target_path, c) is True:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def spec_draft_flags(target_path: str) -> str:
|
||||
"""llama-server-Flags für Speculative Decoding (Draft + --spec-type), oder ''
|
||||
wenn kein kompatibler Draft existiert. Beides nötig, sonst ist Spec inaktiv."""
|
||||
d = find_compatible_draft(target_path)
|
||||
return f" --spec-draft-model {d} --spec-type {SPEC_TYPE}" if d else ""
|
||||
|
||||
|
||||
def drafts_for(target_path: str) -> dict:
|
||||
"""Für die UI: alle Drafts + ihre Kompatibilität zum Ziel-Modell.
|
||||
compatible=None heißt 'nicht prüfbar' (Ziel- oder Draft-GGUF fehlt)."""
|
||||
from services import gguf_meta
|
||||
exists = bool(target_path and os.path.exists(target_path))
|
||||
drafts = list_drafts()
|
||||
for d in drafts:
|
||||
d["compatible"] = gguf_meta.compatible(target_path, d["path"]) if exists else None
|
||||
return {
|
||||
"target_path": target_path,
|
||||
"target_exists": exists,
|
||||
"target_vocab": gguf_meta.fingerprint(target_path) if exists else None,
|
||||
"drafts": drafts,
|
||||
}
|
||||
|
||||
|
||||
def set_spec_draft(model_id: str, draft_path: str | None) -> dict:
|
||||
"""Setzt (oder entfernt mit draft_path=None) den Spec-Draft eines Modells.
|
||||
Validiert die Vocab-Kompatibilität — ein inkompatibler/unprüfbarer Draft wird
|
||||
abgelehnt (idiotensicher). Returns {ok, reason}."""
|
||||
cfg = read_config()
|
||||
spec = (cfg.get("models") or {}).get(model_id)
|
||||
if not isinstance(spec, dict):
|
||||
return {"ok": False, "reason": "Modell nicht gefunden"}
|
||||
cmd = str(spec.get("cmd", ""))
|
||||
# vorhandene Spec-Flags entfernen (idempotent)
|
||||
cmd = re.sub(r"\s+--spec-draft-model\s+\S+", "", cmd)
|
||||
cmd = re.sub(r"\s+--spec-type\s+\S+", "", cmd)
|
||||
|
||||
if draft_path:
|
||||
# relative Angabe (nur Dateiname) gegen DRAFTS_DIR auflösen
|
||||
if not os.path.isabs(draft_path) and "/" not in draft_path:
|
||||
draft_path = str(DRAFTS_DIR / draft_path)
|
||||
if not os.path.exists(draft_path):
|
||||
return {"ok": False, "reason": "Draft-Datei nicht gefunden"}
|
||||
from services import gguf_meta
|
||||
target = ""
|
||||
if (mt := _PATH_RE.search(cmd)):
|
||||
target = mt.group(1).replace("'", "").replace('"', "")
|
||||
comp = gguf_meta.compatible(target, draft_path) if os.path.exists(target) else None
|
||||
if comp is not True:
|
||||
reason = ("Draft ist NICHT vocab-kompatibel zum Modell — Speculative Decoding "
|
||||
"würde beim Laden scheitern."
|
||||
if comp is False else
|
||||
"Kompatibilität nicht prüfbar (Modell-GGUF fehlt) — Draft nicht gesetzt.")
|
||||
return {"ok": False, "reason": reason}
|
||||
cmd = cmd.rstrip() + f" --spec-draft-model {draft_path} --spec-type {SPEC_TYPE}"
|
||||
|
||||
spec["cmd"] = LiteralScalarString(cmd.rstrip() + "\n")
|
||||
write_config(cfg)
|
||||
return {"ok": True, "reason": ""}
|
||||
|
||||
|
||||
# --- Groups (Ko-Residenz) ----------------------------------------------------
|
||||
def set_group(group: str, members: list[str], swap: bool = False, persist: bool = False) -> None:
|
||||
"""llama-swap-`groups`-Eintrag setzen. swap=False → alle Mitglieder dürfen
|
||||
|
||||
Reference in New Issue
Block a user