""" Wartung: Updates (OS/Engine/Modelle), Dienst-Neustart (system- vs user-aware), Reboot, Logs. Portiert/modernisiert aus Mission Control v1 (routers/maintenance.py). Passwortfrei über NOPASSWD-Whitelist (sudo -n). OS-Update/Reboot brauchen einmalig erweiterte sudoers (siehe docs/BEDIENUNG.md). Lange Ops laufen als jobengine-Job. """ import os import re import subprocess import time from datetime import datetime import httpx import psutil from services import catalog, discover, jobengine, llamaswap, system # System-Dienste (root, via sudo -n NOPASSWD) vs. User-Dienste (systemctl --user). SYSTEM_SERVICES = {"llama-swap"} USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-dashboard", "hermes-webui"} # Engine-Update: lädt den neuesten Vulkan-Build (deploy/update-engine.sh, läuft als root). _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) ENGINE_UPDATE_CMD = os.environ.get( "MC_ENGINE_UPDATE_CMD", f"sudo bash {_REPO_ROOT}/deploy/update-engine.sh") # Engine = offizieller Vulkan-Build von ggml-org/llama.cpp (RADV auf Strix Halo). ENGINE_PATH = os.environ.get("MC_ENGINE_PATH", "/opt/llamacpp-vulkan") ENGINE_REPO = os.environ.get("MC_ENGINE_REPO", "ggml-org/llama.cpp") _engine_cache = {"ts": 0.0, "avail": False} def _installed_engine_build() -> int | None: """Build-Nummer der installierten llama-server-Binary (z.B. 9821), oder None. Vulkan-Build braucht LD_LIBRARY_PATH=ENGINE_PATH zum Start von --version.""" bin_path = os.path.join(ENGINE_PATH, "llama-server") if not os.path.exists(bin_path): return None try: env = dict(os.environ, LD_LIBRARY_PATH=ENGINE_PATH) out = subprocess.run([bin_path, "--version"], capture_output=True, text=True, timeout=20, env=env) txt = (out.stderr or "") + (out.stdout or "") if (m := re.search(r"build:\s*\S+\s*\((\d+)\)", txt)) or (m := re.search(r"\bb(\d{3,})\b", txt)): return int(m.group(1)) except Exception: return None return None def _ram_gb() -> float: return psutil.virtual_memory().total / (1024 ** 3) def _os_upgradable() -> int: try: out = subprocess.run( ["bash", "-c", "apt list --upgradable 2>/dev/null | grep -c upgradable || true"], capture_output=True, text=True, timeout=10) return int((out.stdout or "0").strip() or 0) except Exception: return 0 def _engine_update_available() -> bool: now = time.time() if now - _engine_cache["ts"] < 3600: return _engine_cache["avail"] avail = False try: rel = httpx.get(f"https://api.github.com/repos/{ENGINE_REPO}/releases/latest", timeout=6, headers={"User-Agent": "MissionControl2"}).json() tag = str(rel.get("tag_name", "")) latest = int(m.group(1)) if (m := re.search(r"(\d{3,})", tag)) else None installed = _installed_engine_build() if latest is not None and installed is not None: avail = latest > installed # präziser Build-Nummer-Vergleich else: # Fallback: Release-Datum vs. Engine-mtime pub = datetime.fromisoformat(rel["published_at"].replace("Z", "+00:00")).timestamp() avail = pub > os.path.getmtime(ENGINE_PATH) + 86400 except Exception: avail = False _engine_cache.update(ts=now, avail=avail) return avail _comp_cache = {"ts": 0.0, "data": []} def _hermes_agent_update() -> dict: """Hermes-Agent wird aus **git** aktualisiert (CLI `hermes update` = git pull origin ). Darum HEAD vs. origin/ prüfen (fetch + behind-count) — NICHT GitHub-Releases: die werden selten getaggt, main läuft ihnen voraus → sonst zeigt das UI nie ein Update an.""" info = {"key": "hermes_agent", "name": "Hermes Agent", "current": None, "latest": None, "update": False, "reachable": None} git = system.find_hermes_agent_git() if not git or not git.get("path"): return info path = git["path"] info["current"] = git.get("hash") try: branch = (subprocess.run(["git", "-C", path, "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True, timeout=8).stdout.strip() or "main") fetch = subprocess.run(["git", "-C", path, "fetch", "-q", "origin", branch], capture_output=True, text=True, timeout=25) info["reachable"] = (fetch.returncode == 0) if fetch.returncode == 0: cnt = subprocess.run(["git", "-C", path, "rev-list", "--count", f"HEAD..origin/{branch}"], capture_output=True, text=True, timeout=8) behind = int(cnt.stdout.strip() or "0") if cnt.returncode == 0 else 0 info["behind"] = behind info["update"] = behind > 0 oh = subprocess.run(["git", "-C", path, "rev-parse", "--short", f"origin/{branch}"], capture_output=True, text=True, timeout=8).stdout.strip() info["latest"] = (f"{oh} ({behind} neu)" if behind else (oh or info["current"])) except Exception: info["reachable"] = False return info def _components_cached() -> list[dict]: """Update-Status von Hermes-Agent (1h-Cache → GitHub schonen).""" now = time.time() if now - _comp_cache["ts"] < 3600 and _comp_cache["data"]: return _comp_cache["data"] data = [_hermes_agent_update()] _comp_cache.update(ts=now, data=data) return data def _params_of(m: dict) -> float: """Größen-bewusste Parameterzahl eines installierten Modells: max aus Namens-Schätzung und Dateigröße (fängt namenlose wie 'Qwen3-Coder-Next' UND Split-GGUFs ab).""" from services.fit import QUANT_BYTES_PER_PARAM caps = m.get("capabilities") or {} bpp = QUANT_BYTES_PER_PARAM.get((m.get("quant") or "Q4_K_M").upper(), 0.55) size_gb = (m.get("size_bytes") or 0) / (1024 ** 3) pb_size = (size_gb / bpp) if size_gb > 1.0 else 0.0 return max(float(caps.get("params_b") or 0), pb_size, 0.0) # Familien-Subtyp + Generations-Version aus dem Modellnamen (für „echtes Upgrade?"). _FAM_PATS = (("qwen", r"qwen(\d+(?:\.\d+)?)"), ("gemma", r"gemma[-_ ]?(\d+(?:\.\d+)?)"), ("llama", r"llama[-_ ]?(\d+(?:\.\d+)?)"), ("phi", r"phi[-_ ]?(\d+(?:\.\d+)?)"), ("mistral", r"mistral"), ("hermes", r"hermes[-_ ]?(\d+(?:\.\d+)?)")) def _gen_key(name: str): """(Familie+Subtyp, Generations-Version) oder None. Z.B. 'Qwen3-VL-2B' → ('qwen-vl', 3.0), 'Qwen2.5-VL-7B' → ('qwen-vl', 2.5). Nur gleiche Familie ist sinnvoll vergleichbar.""" low = (name or "").lower() sub = "-vl" if any(k in low for k in ("-vl", "vl-", "vision", "llava", "pixtral")) else \ "-coder" if ("coder" in low or "-code" in low) else "" for fam, pat in _FAM_PATS: m = re.search(pat, low) if m: ver = float(m.group(1)) if (m.groups() and m.group(1)) else 0.0 return (fam + sub, ver) return None def _meta(name: str, model_dict: dict | None = None, im: dict | None = None) -> dict: """Metadaten (family, gen, total, active, moe) — bevorzugt den kuratierten Katalog, sonst die Felder eines Discover-/Modell-Dicts, sonst Namens-/Größen-Heuristik.""" cm = catalog.meta_for_name(name) if cm: return {"family": cm.get("family"), "gen": cm.get("generation"), "total": float(cm.get("total_params_b") or 0), "active": cm.get("active_params_b"), "moe": bool(cm.get("moe"))} d = model_dict or {} g = _gen_key(name) total = float(d.get("params_b") or 0) or (_params_of(im) if im else 0.0) return {"family": (d.get("family") or (g[0] if g else None)), "gen": (d.get("generation") if d.get("generation") is not None else (g[1] if g else None)), "total": total, "active": d.get("active_b"), "moe": bool(d.get("moe"))} def model_upgrades() -> list[dict]: """Je Rolle ein ECHTES Upgrade — nur wenn die Empfehlung wirklich besser ist: gleiche Familie UND (neuere Generation ODER deutlich größer) UND kein Tempo-Downgrade (MoE-first für die bandbreiten-limitierte Box: dense ersetzt MoE nur bei großem Wissens- Sprung). Metadaten kommen aus dem kuratierten Katalog → keine Namens-Raterei.""" disc = discover.safe_discover(_ram_gb()) if not disc: return [] installed = llamaswap.list_models() inst_by_role = {m["role"]: m for m in installed if m.get("role")} cmds = " ".join(str(s.get("cmd", "")).lower() for s in (llamaswap.read_config().get("models") or {}).values()) out = [] for c in disc.get("categories", []): role = c["role"] im = inst_by_role.get(role) if im is None: continue rec = c.get("recommended") if not rec: continue rec_model = next((x for x in c.get("models", []) if x.get("repo") == rec), None) i = _meta(im["name"], im=im) r = _meta(rec, model_dict=rec_model) if not i["family"] or not r["family"] or i["family"] != r["family"]: continue # andere/unbekannte Familie → kein Upgrade if r["gen"] is not None and i["gen"] is not None and r["gen"] < i["gen"] - 1e-6: continue # ältere Generation → niemals same_gen = (r["gen"] is None or i["gen"] is None or abs(r["gen"] - i["gen"]) < 1e-6) if same_gen: if r["total"] and i["total"] and r["total"] < i["total"] * 1.05: continue # gleiche Gen, nicht größer → kein Upgrade # MoE-first: ein MoE durch dense ersetzen nur bei deutlichem Wissens-Sprung if i["moe"] and not r["moe"] and r["total"] < i["total"] * 1.5: continue # Tempo nicht verschlechtern (aktive Params), außer großer Wissens-Gewinn ia, ra = (i["active"] or i["total"]), (r["active"] or r["total"]) if ia and ra > ia * 1.3 and r["total"] < i["total"] * 1.3: continue base = rec.split("/")[-1].lower() stem = base[:-5] if base.endswith("-gguf") else base if base in cmds or (stem and stem in cmds): continue # schon installiert out.append({"role": role, "title": c["title"], "repo": rec}) return out def _last_apt_update() -> float | None: for path in ["/var/lib/apt/periodic/update-success-stamp", "/var/cache/apt/pkgcache.bin"]: if os.path.exists(path): try: return os.path.getmtime(path) except Exception: pass return None def updates() -> dict: ups = model_upgrades() return {"os": _os_upgradable(), "engine": 1 if _engine_update_available() else 0, "models": len(ups), "model_list": ups, "last_check": _last_apt_update(), "components": _components_cached()} def _run(cmd: list[str], sudo_password: str | None = None) -> dict: actual_cmd = list(cmd) has_sudo = False if cmd and cmd[0] == "sudo": has_sudo = True # If we have a password, use -S instead of -n if sudo_password is not None: if "-n" in actual_cmd: actual_cmd = [x for x in actual_cmd if x != "-n"] if "-S" not in actual_cmd: actual_cmd.insert(1, "-S") else: # Force -n to fail cleanly if password is required if "-S" in actual_cmd: actual_cmd = [x for x in actual_cmd if x != "-S"] if "-n" not in actual_cmd: actual_cmd.insert(1, "-n") try: input_data = (sudo_password + "\n") if (has_sudo and sudo_password is not None) else None p = subprocess.run(actual_cmd, input=input_data, capture_output=True, text=True, timeout=120) err_msg = p.stderr or "" if p.returncode != 0 and ("a password is required" in err_msg or "password" in err_msg.lower() or "sudo:" in err_msg): if sudo_password is not None: return {"ok": False, "status": "incorrect_password", "out": p.stdout or "", "err": "Falsches Sudo-Passwort."} return {"ok": False, "status": "password_required", "out": p.stdout or "", "err": "Sudo-Passwort erforderlich."} return {"ok": p.returncode == 0, "out": (p.stdout or "")[-4000:], "err": (p.stderr or "")[-2000:]} except Exception as exc: # noqa: BLE001 return {"ok": False, "out": "", "err": str(exc)} def check_sudo_needs_password(sudo_password: str | None = None) -> dict | None: """Checks if sudo needs a password. Returns error dict if password required/incorrect, else None.""" res = _run(["sudo", "true"], sudo_password=sudo_password) if not res["ok"]: return res return None def restart_service(name: str, sudo_password: str | None = None) -> dict: if name in SYSTEM_SERVICES: if err := check_sudo_needs_password(sudo_password): return err return _run(["sudo", "systemctl", "restart", name], sudo_password=sudo_password) if name in USER_SERVICES: return _run(["systemctl", "--user", "restart", name]) return {"ok": False, "err": f"Dienst '{name}' nicht erlaubt."} def logs(service: str, lines: int = 200, sudo_password: str | None = None) -> dict: lines = max(1, min(lines, 1000)) if service in USER_SERVICES: r = _run(["journalctl", "--user", "-u", service, "-n", str(lines), "--no-pager"]) return {"ok": r["ok"], "text": r["out"] or r["err"]} if service in SYSTEM_SERVICES: # Journal-Lesen braucht i.d.R. KEIN sudo (User ist in Gruppe adm/systemd-journal). # Erst ohne sudo versuchen; nur bei fehlenden Rechten auf sudo zurückfallen. r = _run(["journalctl", "-u", service, "-n", str(lines), "--no-pager"]) if r["ok"]: return {"ok": True, "text": r["out"] or "(keine Log-Einträge)"} if err := check_sudo_needs_password(sudo_password): return err r = _run(["sudo", "journalctl", "-u", service, "-n", str(lines), "--no-pager"], sudo_password=sudo_password) return {"ok": r["ok"], "text": r["out"] or r["err"]} return {"ok": False, "text": "", "err": "Dienst nicht erlaubt."} def check_updates_job(sudo_password: str | None = None) -> dict: if err := check_sudo_needs_password(sudo_password): return err def on_done(): _engine_cache.update(ts=0.0, avail=False) cmd = "sudo apt-get update" job_id = jobengine.start_job(["bash", "-c", cmd], "Nach Updates suchen", on_done=on_done, sudo_password=sudo_password) return {"ok": True, "job_id": job_id} def os_update_job(sudo_password: str | None = None) -> dict: if err := check_sudo_needs_password(sudo_password): return err cmd = "sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y" job_id = jobengine.start_job(["bash", "-c", cmd], "OS-Update (apt)", sudo_password=sudo_password) return {"ok": True, "job_id": job_id} def engine_update_job(sudo_password: str | None = None) -> dict | None: if not ENGINE_UPDATE_CMD: return None if err := check_sudo_needs_password(sudo_password): return err # update-engine.sh läuft via sudo als root und startet llama-swap am Ende selbst neu. job_id = jobengine.start_job(["bash", "-c", ENGINE_UPDATE_CMD], "Engine-Update (llama.cpp Vulkan)", sudo_password=sudo_password) return {"ok": True, "job_id": job_id} def reboot(sudo_password: str | None = None) -> dict: if err := check_sudo_needs_password(sudo_password): return err return _run(["sudo", "reboot"], sudo_password=sudo_password)