Feat: Vulkan/RADV-Engine + vocab-gepruefte Spec-Drafts + Provisioning/Sync
Engine-Cutover ROCm/HIP -> Vulkan/RADV (gfx1151): +12-22% tg auf MoE (llama-bench
verifiziert, fast 53->65 t/s). ROCm-Build bleibt als Rollback unter /opt/llamacpp.
- Backend: vocab-aware Speculative Decoding. services/gguf_meta.py liest den
Tokenizer-Fingerprint (model/pre/n_vocab) direkt aus dem GGUF-Header (ohne Modell-Load);
register_model + migrate_config haengen nur VOCAB-KOMPATIBLE Drafts an (inkl. --spec-type,
das in dieser llama.cpp-Generation noetig ist). Neue Endpoints /api/models/drafts + /{id}/draft.
- Frontend: idiotensichere Spec-Draft-UI (SpecDraftModal) - nur kompatible Drafts waehlbar,
inkompatible gesperrt mit Begruendung; SPEC/SPEC?-Badge nach echtem Aktiv-Status; Rolle in AddModel.
- maintenance.py: Engine-Update-Quelle -> ggml-org/llama.cpp (Build-Nummer-Vergleich),
ENGINE_PATH=/opt/llamacpp-vulkan.
- Startup-Warmup der brains (deploy/warmup.sh, self-detaching ExecStartPost) + deploy/provision-engine.sh.
- Cleanup: tote LiteLLM gateway/config.yaml + alle Referenzen (config.py/backup.py/backup.sh) entfernt;
README + docs/memory aktualisiert.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
+14
-7
@@ -33,17 +33,24 @@ CMD_TEMPLATE = os.environ.get("MC_CMD_TEMPLATE", _DEFAULT_CMD_TEMPLATE)
|
||||
if "{model}" not in CMD_TEMPLATE:
|
||||
CMD_TEMPLATE = _DEFAULT_CMD_TEMPLATE
|
||||
DEFAULT_TTL = int(os.environ.get("MC_DEFAULT_TTL", "300"))
|
||||
# Draft-Modell für Speculative Decoding (nur fast/coder, wenn vorhanden). Eine
|
||||
# Quelle der Wahrheit für llamaswap.register_model + migrate_config.
|
||||
SPEC_DRAFT_MODEL_PATH = os.environ.get(
|
||||
"MC_SPEC_DRAFT_MODEL", f"{MODELS_DIR.as_posix()}/drafts/qwen2.5-1.5b-instruct-q4_k_m.gguf")
|
||||
# Verzeichnis mit Draft-Modellen für Speculative Decoding. Beim Hinzufügen eines
|
||||
# fast/coder-Modells wird hieraus automatisch ein **vocab-kompatibler** Draft gewählt
|
||||
# (Vocab-Check via services.gguf_meta; ein inkompatibler Draft lässt llama.cpp scheitern).
|
||||
DRAFTS_DIR = Path(os.environ.get("MC_DRAFTS_DIR", str(MODELS_DIR / "drafts")))
|
||||
# Optionaler expliziter Default-Draft (leer = Auto-Erkennung aus DRAFTS_DIR). Wird nur
|
||||
# verwendet, wenn er zum Ziel-Modell vocab-kompatibel ist. (Früher fix qwen2.5 → entfernt,
|
||||
# weil das mit neueren Vocabs wie Qwen3.6 inkompatibel ist und Spec stillschweigend brach.)
|
||||
SPEC_DRAFT_MODEL_PATH = os.environ.get("MC_SPEC_DRAFT_MODEL", "")
|
||||
# Speculative-Decoding-Typ (llama.cpp dieser Generation braucht --spec-type zusätzlich
|
||||
# zu --spec-draft-model, sonst ist Spec inaktiv).
|
||||
SPEC_TYPE = os.environ.get("MC_SPEC_TYPE", "draft-simple")
|
||||
# Env für HuggingFace-Downloads: XET deaktivieren (Hänger bei ~6 MB, siehe v1-Gotcha).
|
||||
HF_DOWNLOAD_ENV = {"HF_HUB_DISABLE_XET": "1"}
|
||||
|
||||
# --- Routing-Gateway (LiteLLM, model: auto) ----------------------------------
|
||||
# --- Routing-Gateway (builtin in MC2, model: auto) ---------------------------
|
||||
# MC2 IST der Gateway (services/gateway.py + routers/gateway_proxy.py). KEIN externer
|
||||
# LiteLLM-Dienst (scheitert auf Python 3.14). Daher keine Gateway-Config-Datei mehr.
|
||||
GATEWAY_URL = os.environ.get("MC_GATEWAY_URL", f"http://127.0.0.1:{os.environ.get('MC_PORT', '9000')}").rstrip("/")
|
||||
GATEWAY_CONFIG_PATH = Path(os.environ.get(
|
||||
"MC_GATEWAY_CONFIG", str(Path(__file__).resolve().parent.parent / "gateway" / "config.yaml")))
|
||||
|
||||
# --- Hermes Agent (eigener Dienst auf der Box) -------------------------------
|
||||
# Gateway (OpenAI-API des Agenten) + standalone Web-UI (nesquena/hermes-webui).
|
||||
|
||||
@@ -4,8 +4,8 @@ from pathlib import Path
|
||||
# Add backend directory to sys.path so we can import services
|
||||
sys.path.append(str(Path(__file__).resolve().parent))
|
||||
|
||||
from services.llamaswap import read_config, write_config
|
||||
from config import CONFIG_PATH, SPEC_DRAFT_MODEL_PATH
|
||||
from services.llamaswap import read_config, write_config, spec_draft_flags, _PATH_RE
|
||||
from config import CONFIG_PATH
|
||||
|
||||
def migrate():
|
||||
print(f"Reading config from {CONFIG_PATH}...")
|
||||
@@ -16,8 +16,6 @@ def migrate():
|
||||
cfg = read_config()
|
||||
models = cfg.get("models", {})
|
||||
|
||||
draft_path = SPEC_DRAFT_MODEL_PATH
|
||||
|
||||
for name, spec in models.items():
|
||||
if not isinstance(spec, dict):
|
||||
continue
|
||||
@@ -36,12 +34,15 @@ def migrate():
|
||||
aliases = spec.get("aliases", [])
|
||||
role = aliases[0] if aliases else None
|
||||
|
||||
# 3. Add parallel and speculative decoding for fast and coder
|
||||
# 3. Add parallel + (nur vocab-kompatibles) Speculative Decoding für fast/coder.
|
||||
# spec_draft_flags() prüft die Vocab-Kompatibilität und hängt --spec-type an;
|
||||
# ein inkompatibler Draft (z.B. qwen2.5 ↔ Qwen3.6) wird NICHT gesetzt.
|
||||
if role in ("fast", "coder"):
|
||||
if "--parallel" not in cmd:
|
||||
cmd = cmd.strip() + " --parallel 2"
|
||||
if "--spec-draft-model" not in cmd:
|
||||
cmd = cmd.strip() + f" --spec-draft-model {draft_path}"
|
||||
target = mt.group(1) if (mt := _PATH_RE.search(cmd)) else ""
|
||||
cmd = cmd.strip() + spec_draft_flags(target)
|
||||
|
||||
# Update cmd
|
||||
from ruamel.yaml.scalarstring import LiteralScalarString
|
||||
|
||||
@@ -158,6 +158,30 @@ def set_model_ctx(model_id: str, body: CtxReq) -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/models/drafts")
|
||||
def list_drafts(target: str = "") -> dict:
|
||||
"""Verfügbare Draft-Modelle + ihre Vocab-Kompatibilität zum Ziel-Modell
|
||||
(target = GGUF-Pfad). Basis für die idiotensichere Spec-Draft-Auswahl im UI."""
|
||||
return llamaswap.drafts_for(target)
|
||||
|
||||
|
||||
class DraftReq(BaseModel):
|
||||
draft_path: str | None = None
|
||||
|
||||
|
||||
@router.post("/models/{model_id}/draft")
|
||||
def set_model_draft(model_id: str, body: DraftReq) -> dict:
|
||||
"""Setzt/entfernt den Speculative-Decoding-Draft eines Modells. Inkompatible
|
||||
(oder nicht prüfbare) Drafts werden serverseitig abgelehnt."""
|
||||
try:
|
||||
res = llamaswap.set_spec_draft(model_id, body.draft_path)
|
||||
except PermissionError as exc:
|
||||
raise HTTPException(500, str(exc))
|
||||
if not res["ok"]:
|
||||
raise HTTPException(400 if "kompatib" in res["reason"].lower() else 404, res["reason"])
|
||||
return res
|
||||
|
||||
|
||||
@router.post("/models/unload")
|
||||
def unload_all_models() -> dict:
|
||||
import httpx
|
||||
|
||||
@@ -7,7 +7,7 @@ import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from config import CONFIG_PATH, GATEWAY_CONFIG_PATH, MEMORY_DB, MODELS_DIR
|
||||
from config import CONFIG_PATH, MEMORY_DB, MODELS_DIR
|
||||
|
||||
BACKUP_DIR = Path(MODELS_DIR) / "mc2-backups"
|
||||
RETAIN = 7
|
||||
@@ -31,7 +31,7 @@ def backup_now() -> dict:
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
saved = []
|
||||
for src in (MEMORY_DB, Path(str(MEMORY_DB) + "-wal"), Path(str(MEMORY_DB) + "-shm"),
|
||||
CONFIG_PATH, GATEWAY_CONFIG_PATH):
|
||||
CONFIG_PATH):
|
||||
if (name := _safe_copy(src, dst)):
|
||||
saved.append(name)
|
||||
# Aufräumen: nur die letzten RETAIN Snapshots behalten.
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
GGUF-Tokenizer-Fingerprint — liest die Tokenizer-Identität direkt aus dem
|
||||
GGUF-Header (ohne das Modell zu laden), um zu entscheiden, ob ein Draft-Modell
|
||||
**vocab-kompatibel** mit einem Ziel-Modell ist (Voraussetzung für Speculative
|
||||
Decoding in llama.cpp — sonst: "draft model vocab type must match target").
|
||||
|
||||
Wir lesen nur die Metadaten-KV-Sektion am Dateianfang und brechen ab, sobald
|
||||
`tokenizer.ggml.tokens` erreicht ist (dessen Länge = n_vocab). model+pre+n_vocab
|
||||
identifizieren den Tokenizer eindeutig genug, um die in der Praxis relevanten
|
||||
Fälle zu unterscheiden (Qwen2.5 vs Qwen3 vs Qwen3.6 etc.). Die llama.cpp-Prüfung
|
||||
beim Laden bleibt der letzte Schiedsrichter.
|
||||
"""
|
||||
|
||||
import struct
|
||||
from functools import lru_cache
|
||||
|
||||
# GGUF value types (https://github.com/ggml-org/ggml/blob/master/docs/gguf.md)
|
||||
_T_UINT8, _T_INT8, _T_UINT16, _T_INT16, _T_UINT32, _T_INT32, _T_FLOAT32, \
|
||||
_T_BOOL, _T_STRING, _T_ARRAY, _T_UINT64, _T_INT64, _T_FLOAT64 = range(13)
|
||||
|
||||
_SCALAR_FMT = {
|
||||
_T_UINT8: "<B", _T_INT8: "<b", _T_UINT16: "<H", _T_INT16: "<h",
|
||||
_T_UINT32: "<I", _T_INT32: "<i", _T_FLOAT32: "<f", _T_BOOL: "<?",
|
||||
_T_UINT64: "<Q", _T_INT64: "<q", _T_FLOAT64: "<d",
|
||||
}
|
||||
_SCALAR_SIZE = {t: struct.calcsize(f) for t, f in _SCALAR_FMT.items()}
|
||||
|
||||
_WANT_STRINGS = {"tokenizer.ggml.model", "tokenizer.ggml.pre", "general.architecture"}
|
||||
|
||||
|
||||
class _Reader:
|
||||
def __init__(self, f):
|
||||
self.f = f
|
||||
|
||||
def read(self, n: int) -> bytes:
|
||||
b = self.f.read(n)
|
||||
if len(b) != n:
|
||||
raise EOFError("unerwartetes Dateiende beim GGUF-Parsen")
|
||||
return b
|
||||
|
||||
def u32(self) -> int:
|
||||
return struct.unpack("<I", self.read(4))[0]
|
||||
|
||||
def u64(self) -> int:
|
||||
return struct.unpack("<Q", self.read(8))[0]
|
||||
|
||||
def gstr(self) -> str:
|
||||
n = self.u64()
|
||||
return self.read(n).decode("utf-8", "replace")
|
||||
|
||||
def skip_value(self, vtype: int) -> None:
|
||||
"""Liest einen Wert und verwirft ihn (um den Datei-Pointer korrekt
|
||||
weiterzuschieben). Arrays werden elementweise konsumiert."""
|
||||
if vtype == _T_STRING:
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif vtype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[vtype], 1)
|
||||
elif vtype == _T_ARRAY:
|
||||
etype = self.u32()
|
||||
count = self.u64()
|
||||
if etype == _T_STRING:
|
||||
for _ in range(count):
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif etype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[etype] * count, 1)
|
||||
else:
|
||||
raise ValueError(f"unbekannter Array-Elementtyp {etype}")
|
||||
else:
|
||||
raise ValueError(f"unbekannter GGUF-Wertetyp {vtype}")
|
||||
|
||||
|
||||
def _read_fingerprint(path: str) -> dict | None:
|
||||
"""Liest model/pre/n_vocab aus dem GGUF-Header. None bei Fehler/kein GGUF."""
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
r = _Reader(fh)
|
||||
if r.read(4) != b"GGUF":
|
||||
return None
|
||||
r.u32() # version
|
||||
r.u64() # tensor_count
|
||||
kv_count = r.u64()
|
||||
fp: dict = {"model": None, "pre": None, "arch": None, "n_vocab": None}
|
||||
for _ in range(kv_count):
|
||||
key = r.gstr()
|
||||
vtype = r.u32()
|
||||
if key == "tokenizer.ggml.tokens" and vtype == _T_ARRAY:
|
||||
etype = r.u32()
|
||||
fp["n_vocab"] = r.u64()
|
||||
# Wir haben alles (model/pre kommen vor tokens) → abbrechen.
|
||||
if etype != _T_STRING:
|
||||
return None
|
||||
break
|
||||
if key in _WANT_STRINGS and vtype == _T_STRING:
|
||||
val = r.gstr()
|
||||
if key == "tokenizer.ggml.model":
|
||||
fp["model"] = val
|
||||
elif key == "tokenizer.ggml.pre":
|
||||
fp["pre"] = val
|
||||
else:
|
||||
fp["arch"] = val
|
||||
else:
|
||||
r.skip_value(vtype)
|
||||
if fp["model"] is None and fp["n_vocab"] is None:
|
||||
return None
|
||||
return fp
|
||||
except (OSError, EOFError, ValueError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _cached(path: str, mtime: float, size: int) -> tuple | None:
|
||||
fp = _read_fingerprint(path)
|
||||
if fp is None:
|
||||
return None
|
||||
return (fp.get("model"), fp.get("pre"), fp.get("n_vocab"), fp.get("arch"))
|
||||
|
||||
|
||||
def fingerprint(path: str) -> dict | None:
|
||||
"""Tokenizer-Fingerprint eines GGUF (gecacht nach Pfad+mtime+size).
|
||||
Returns dict(model, pre, n_vocab, arch) oder None wenn nicht lesbar."""
|
||||
import os
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
t = _cached(path, st.st_mtime, st.st_size)
|
||||
if t is None:
|
||||
return None
|
||||
return {"model": t[0], "pre": t[1], "n_vocab": t[2], "arch": t[3]}
|
||||
|
||||
|
||||
def vocab_key(path: str) -> tuple | None:
|
||||
"""Vergleichsschlüssel für Vocab-Kompatibilität: (model, pre, n_vocab).
|
||||
Genau diese Identität verlangt llama.cpp für Speculative Decoding."""
|
||||
fp = fingerprint(path)
|
||||
if not fp or fp["n_vocab"] is None:
|
||||
return None
|
||||
return (fp["model"], fp["pre"], fp["n_vocab"])
|
||||
|
||||
|
||||
def compatible(target_path: str, draft_path: str) -> bool | None:
|
||||
"""True/False ob draft vocab-kompatibel zum target ist. None = unbestimmbar
|
||||
(eine Datei nicht lesbar) → UI behandelt das als 'nicht bestätigt'."""
|
||||
a = vocab_key(target_path)
|
||||
b = vocab_key(draft_path)
|
||||
if a is None or b is None:
|
||||
return None
|
||||
return a == b
|
||||
@@ -13,7 +13,10 @@ import re
|
||||
import httpx
|
||||
from ruamel.yaml.scalarstring import LiteralScalarString
|
||||
|
||||
from config import CMD_TEMPLATE, CONFIG_PATH, DEFAULT_TTL, LLAMA_SWAP_URL, SPEC_DRAFT_MODEL_PATH
|
||||
from config import (
|
||||
CMD_TEMPLATE, CONFIG_PATH, DEFAULT_TTL, DRAFTS_DIR, LLAMA_SWAP_URL,
|
||||
SPEC_DRAFT_MODEL_PATH, SPEC_TYPE,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -66,6 +69,11 @@ def _parse_model(name: str, spec: dict) -> dict:
|
||||
m_draft = re.search(r"--spec-draft-model\s+([^\s]+)", cmd)
|
||||
if m_draft:
|
||||
spec_draft = os.path.basename(m_draft.group(1).replace("'", "").replace('"', ""))
|
||||
spec_type = None
|
||||
if (m_st := re.search(r"--spec-type\s+([^\s]+)", cmd)):
|
||||
spec_type = m_st.group(1)
|
||||
# Spec ist nur AKTIV, wenn BEIDES gesetzt ist (--spec-draft-model UND --spec-type).
|
||||
spec_active = bool(spec_draft and spec_type)
|
||||
parallel_match = re.search(r"--parallel\s+(\d+)", cmd)
|
||||
parallel_slots = int(parallel_match.group(1)) if parallel_match else 1
|
||||
|
||||
@@ -85,6 +93,8 @@ def _parse_model(name: str, spec: dict) -> dict:
|
||||
"incomplete": not path,
|
||||
"prompt_cache": prompt_cache,
|
||||
"spec_draft_model": spec_draft,
|
||||
"spec_type": spec_type,
|
||||
"spec_active": spec_active,
|
||||
"parallel_slots": parallel_slots,
|
||||
"capabilities": capabilities(
|
||||
name=filename or name, cmd=cmd,
|
||||
@@ -191,8 +201,10 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
|
||||
if role_lower in ("fast", "coder"):
|
||||
if "--parallel" not in cmd:
|
||||
cmd += " --parallel 2"
|
||||
if os.path.exists(SPEC_DRAFT_MODEL_PATH) and "--spec-draft-model" not in cmd:
|
||||
cmd += f" --spec-draft-model {SPEC_DRAFT_MODEL_PATH}"
|
||||
# Nur einen VOCAB-KOMPATIBLEN Draft anhängen (sonst scheitert llama.cpp).
|
||||
# Bei frischem Install existiert die GGUF noch nicht → kein Draft (später im UI setzbar).
|
||||
if "--spec-draft-model" not in cmd:
|
||||
cmd += spec_draft_flags(model_path)
|
||||
|
||||
cfg.setdefault("models", {})[model_id] = {
|
||||
"cmd": LiteralScalarString(cmd + "\n"),
|
||||
@@ -203,6 +215,101 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
|
||||
return model_id
|
||||
|
||||
|
||||
# --- Speculative-Draft / Vocab-Kompatibilität --------------------------------
|
||||
def list_drafts() -> list[dict]:
|
||||
"""Alle Draft-GGUFs in DRAFTS_DIR mit Tokenizer-Fingerprint."""
|
||||
from services import gguf_meta
|
||||
out = []
|
||||
if DRAFTS_DIR.is_dir():
|
||||
for p in sorted(DRAFTS_DIR.glob("*.gguf")):
|
||||
out.append({
|
||||
"path": str(p), "filename": p.name,
|
||||
"size_bytes": p.stat().st_size if p.exists() else None,
|
||||
"vocab": gguf_meta.fingerprint(str(p)),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def find_compatible_draft(target_path: str) -> str | None:
|
||||
"""Pfad eines vocab-kompatiblen Drafts für target_path, oder None.
|
||||
Bevorzugt MC_SPEC_DRAFT_MODEL (falls gesetzt+kompatibel), sonst der erste
|
||||
kompatible Draft in DRAFTS_DIR. None auch, wenn target (noch) fehlt (dann
|
||||
nicht verifizierbar → bewusst KEIN Draft anhängen)."""
|
||||
if not target_path or not os.path.exists(target_path):
|
||||
return None
|
||||
from services import gguf_meta
|
||||
candidates: list[str] = []
|
||||
if SPEC_DRAFT_MODEL_PATH and os.path.exists(SPEC_DRAFT_MODEL_PATH):
|
||||
candidates.append(SPEC_DRAFT_MODEL_PATH)
|
||||
for d in list_drafts():
|
||||
if d["path"] not in candidates:
|
||||
candidates.append(d["path"])
|
||||
for c in candidates:
|
||||
if gguf_meta.compatible(target_path, c) is True:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def spec_draft_flags(target_path: str) -> str:
|
||||
"""llama-server-Flags für Speculative Decoding (Draft + --spec-type), oder ''
|
||||
wenn kein kompatibler Draft existiert. Beides nötig, sonst ist Spec inaktiv."""
|
||||
d = find_compatible_draft(target_path)
|
||||
return f" --spec-draft-model {d} --spec-type {SPEC_TYPE}" if d else ""
|
||||
|
||||
|
||||
def drafts_for(target_path: str) -> dict:
|
||||
"""Für die UI: alle Drafts + ihre Kompatibilität zum Ziel-Modell.
|
||||
compatible=None heißt 'nicht prüfbar' (Ziel- oder Draft-GGUF fehlt)."""
|
||||
from services import gguf_meta
|
||||
exists = bool(target_path and os.path.exists(target_path))
|
||||
drafts = list_drafts()
|
||||
for d in drafts:
|
||||
d["compatible"] = gguf_meta.compatible(target_path, d["path"]) if exists else None
|
||||
return {
|
||||
"target_path": target_path,
|
||||
"target_exists": exists,
|
||||
"target_vocab": gguf_meta.fingerprint(target_path) if exists else None,
|
||||
"drafts": drafts,
|
||||
}
|
||||
|
||||
|
||||
def set_spec_draft(model_id: str, draft_path: str | None) -> dict:
|
||||
"""Setzt (oder entfernt mit draft_path=None) den Spec-Draft eines Modells.
|
||||
Validiert die Vocab-Kompatibilität — ein inkompatibler/unprüfbarer Draft wird
|
||||
abgelehnt (idiotensicher). Returns {ok, reason}."""
|
||||
cfg = read_config()
|
||||
spec = (cfg.get("models") or {}).get(model_id)
|
||||
if not isinstance(spec, dict):
|
||||
return {"ok": False, "reason": "Modell nicht gefunden"}
|
||||
cmd = str(spec.get("cmd", ""))
|
||||
# vorhandene Spec-Flags entfernen (idempotent)
|
||||
cmd = re.sub(r"\s+--spec-draft-model\s+\S+", "", cmd)
|
||||
cmd = re.sub(r"\s+--spec-type\s+\S+", "", cmd)
|
||||
|
||||
if draft_path:
|
||||
# relative Angabe (nur Dateiname) gegen DRAFTS_DIR auflösen
|
||||
if not os.path.isabs(draft_path) and "/" not in draft_path:
|
||||
draft_path = str(DRAFTS_DIR / draft_path)
|
||||
if not os.path.exists(draft_path):
|
||||
return {"ok": False, "reason": "Draft-Datei nicht gefunden"}
|
||||
from services import gguf_meta
|
||||
target = ""
|
||||
if (mt := _PATH_RE.search(cmd)):
|
||||
target = mt.group(1).replace("'", "").replace('"', "")
|
||||
comp = gguf_meta.compatible(target, draft_path) if os.path.exists(target) else None
|
||||
if comp is not True:
|
||||
reason = ("Draft ist NICHT vocab-kompatibel zum Modell — Speculative Decoding "
|
||||
"würde beim Laden scheitern."
|
||||
if comp is False else
|
||||
"Kompatibilität nicht prüfbar (Modell-GGUF fehlt) — Draft nicht gesetzt.")
|
||||
return {"ok": False, "reason": reason}
|
||||
cmd = cmd.rstrip() + f" --spec-draft-model {draft_path} --spec-type {SPEC_TYPE}"
|
||||
|
||||
spec["cmd"] = LiteralScalarString(cmd.rstrip() + "\n")
|
||||
write_config(cfg)
|
||||
return {"ok": True, "reason": ""}
|
||||
|
||||
|
||||
# --- Groups (Ko-Residenz) ----------------------------------------------------
|
||||
def set_group(group: str, members: list[str], swap: bool = False, persist: bool = False) -> None:
|
||||
"""llama-swap-`groups`-Eintrag setzen. swap=False → alle Mitglieder dürfen
|
||||
|
||||
@@ -7,6 +7,7 @@ erweiterte sudoers (siehe docs/BEDIENUNG.md). Lange Ops laufen als jobengine-Job
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
@@ -21,10 +22,30 @@ SYSTEM_SERVICES = {"llama-swap"}
|
||||
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-dashboard", "hermes-webui"}
|
||||
|
||||
ENGINE_UPDATE_CMD = os.environ.get("MC_ENGINE_UPDATE_CMD", "")
|
||||
ENGINE_PATH = os.environ.get("MC_ENGINE_PATH", "/opt/llamacpp")
|
||||
# Engine = offizieller Vulkan-Build von ggml-org/llama.cpp (RADV auf Strix Halo).
|
||||
ENGINE_PATH = os.environ.get("MC_ENGINE_PATH", "/opt/llamacpp-vulkan")
|
||||
ENGINE_REPO = os.environ.get("MC_ENGINE_REPO", "ggml-org/llama.cpp")
|
||||
_engine_cache = {"ts": 0.0, "avail": False}
|
||||
|
||||
|
||||
def _installed_engine_build() -> int | None:
|
||||
"""Build-Nummer der installierten llama-server-Binary (z.B. 9821), oder None.
|
||||
Vulkan-Build braucht LD_LIBRARY_PATH=ENGINE_PATH zum Start von --version."""
|
||||
bin_path = os.path.join(ENGINE_PATH, "llama-server")
|
||||
if not os.path.exists(bin_path):
|
||||
return None
|
||||
try:
|
||||
env = dict(os.environ, LD_LIBRARY_PATH=ENGINE_PATH)
|
||||
out = subprocess.run([bin_path, "--version"], capture_output=True, text=True,
|
||||
timeout=20, env=env)
|
||||
txt = (out.stderr or "") + (out.stdout or "")
|
||||
if (m := re.search(r"build:\s*\S+\s*\((\d+)\)", txt)) or (m := re.search(r"\bb(\d{3,})\b", txt)):
|
||||
return int(m.group(1))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _ram_gb() -> float:
|
||||
return psutil.virtual_memory().total / (1024 ** 3)
|
||||
|
||||
@@ -45,11 +66,16 @@ def _engine_update_available() -> bool:
|
||||
return _engine_cache["avail"]
|
||||
avail = False
|
||||
try:
|
||||
rel = httpx.get("https://api.github.com/repos/lemonade-sdk/llamacpp-rocm/releases/latest",
|
||||
rel = httpx.get(f"https://api.github.com/repos/{ENGINE_REPO}/releases/latest",
|
||||
timeout=6, headers={"User-Agent": "MissionControl2"}).json()
|
||||
pub = datetime.fromisoformat(rel["published_at"].replace("Z", "+00:00")).timestamp()
|
||||
inst = os.path.getmtime(ENGINE_PATH)
|
||||
avail = pub > inst + 86400
|
||||
tag = str(rel.get("tag_name", ""))
|
||||
latest = int(m.group(1)) if (m := re.search(r"(\d{3,})", tag)) else None
|
||||
installed = _installed_engine_build()
|
||||
if latest is not None and installed is not None:
|
||||
avail = latest > installed # präziser Build-Nummer-Vergleich
|
||||
else: # Fallback: Release-Datum vs. Engine-mtime
|
||||
pub = datetime.fromisoformat(rel["published_at"].replace("Z", "+00:00")).timestamp()
|
||||
avail = pub > os.path.getmtime(ENGINE_PATH) + 86400
|
||||
except Exception:
|
||||
avail = False
|
||||
_engine_cache.update(ts=now, avail=avail)
|
||||
|
||||
Reference in New Issue
Block a user