mission-control-v2/backend/services/maintenance.py

"""
Wartung: Updates (OS/Engine/Modelle), Dienst-Neustart (system- vs user-aware),
Reboot, Logs. Portiert/modernisiert aus Mission Control v1 (routers/maintenance.py).

Passwortfrei über NOPASSWD-Whitelist (sudo -n). OS-Update/Reboot brauchen einmalig
erweiterte sudoers (siehe docs/BEDIENUNG.md). Lange Ops laufen als jobengine-Job.
"""

import os
import re
import subprocess
import time
from datetime import datetime

import httpx
import psutil

from services import catalog, discover, jobengine, llamaswap, system

# System-Dienste (root, via sudo -n NOPASSWD) vs. User-Dienste (systemctl --user).
SYSTEM_SERVICES = {"llama-swap"}
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service", "voice-service"}

# Engine-Update: lädt den neuesten Vulkan-Build (deploy/update-engine.sh, läuft als root).
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
ENGINE_UPDATE_CMD = os.environ.get(
    "MC_ENGINE_UPDATE_CMD", f"sudo bash {_REPO_ROOT}/deploy/update-engine.sh")
# Engine = offizieller Vulkan-Build von ggml-org/llama.cpp (RADV auf Strix Halo).
ENGINE_PATH = os.environ.get("MC_ENGINE_PATH", "/opt/llamacpp-vulkan")
ENGINE_REPO = os.environ.get("MC_ENGINE_REPO", "ggml-org/llama.cpp")
_engine_cache = {"ts": 0.0, "avail": False}

# Router = llama-swap (mostlygeek): proxyt Anfragen und wechselt die Modelle heiß. Eigenes
# Upstream-Projekt mit eigenem Release-Zyklus → getrennt von der Engine geführt.
SWAP_UPDATE_CMD = os.environ.get(
    "MC_SWAP_UPDATE_CMD", f"sudo bash {_REPO_ROOT}/deploy/update-swap.sh")
SWAP_BIN = os.environ.get("MC_SWAP_BIN", "/usr/local/bin/llama-swap")
SWAP_REPO = os.environ.get("MC_SWAP_REPO", "mostlygeek/llama-swap")
_swap_cache = {"ts": 0.0, "avail": False}

# Stack-Funktionsprüfung NACH jedem Update (OS/Engine/Router): verifiziert per echter
# Inferenz, dass der Stack noch läuft → Job wird rot, wenn ein Update etwas zerschossen hat.
STACK_POSTCHECK = os.path.join(_REPO_ROOT, "deploy", "stack-postcheck.sh")


def _installed_engine_build() -> int | None:
    """Build-Nummer der installierten llama-server-Binary (z.B. 9821), oder None.
    Vulkan-Build braucht LD_LIBRARY_PATH=ENGINE_PATH zum Start von --version."""
    bin_path = os.path.join(ENGINE_PATH, "llama-server")
    if not os.path.exists(bin_path):
        return None
    try:
        env = dict(os.environ, LD_LIBRARY_PATH=ENGINE_PATH)
        out = subprocess.run([bin_path, "--version"], capture_output=True, text=True,
                             timeout=20, env=env)
        txt = (out.stderr or "") + (out.stdout or "")
        # Formate je nach Build: "version: 9821 (hash)" (aktuell), "build: <hash> (9821)", "b9821".
        if (m := re.search(r"version:\s*(\d{3,})", txt)) \
                or (m := re.search(r"build:\s*\S+\s*\((\d+)\)", txt)) \
                or (m := re.search(r"\bb(\d{3,})\b", txt)):
            return int(m.group(1))
    except Exception:
        return None
    return None


def _ram_gb() -> float:
    return psutil.virtual_memory().total / (1024 ** 3)


def _os_upgradable() -> int:
    try:
        # LC_ALL=C erzwingt englische apt-Ausgabe ("[upgradable from: ...]") — sonst zählt
        # grep auf einer deutschen Box ("[aktualisierbar von:]") nichts und meldet faelschlich 0.
        out = subprocess.run(
            ["bash", "-c", "LC_ALL=C apt list --upgradable 2>/dev/null | grep -c upgradable || true"],
            capture_output=True, text=True, timeout=10)
        return int((out.stdout or "0").strip() or 0)
    except Exception:
        return 0


def _engine_update_available() -> bool:
    now = time.time()
    if now - _engine_cache["ts"] < 3600:
        return _engine_cache["avail"]
    avail = False
    try:
        rel = httpx.get(f"https://api.github.com/repos/{ENGINE_REPO}/releases/latest",
                        timeout=6, headers={"User-Agent": "MissionControl2"}).json()
        tag = str(rel.get("tag_name", ""))
        latest = int(m.group(1)) if (m := re.search(r"(\d{3,})", tag)) else None
        installed = _installed_engine_build()
        if latest is not None and installed is not None:
            avail = latest > installed          # präziser Build-Nummer-Vergleich
        else:                                   # Fallback: Release-Datum vs. Engine-mtime
            pub = datetime.fromisoformat(rel["published_at"].replace("Z", "+00:00")).timestamp()
            avail = pub > os.path.getmtime(ENGINE_PATH) + 86400
    except Exception:
        avail = False
    _engine_cache.update(ts=now, avail=avail)
    return avail


def _installed_swap_version() -> int | None:
    """Versions-Nummer der installierten llama-swap-Binary (z.B. 228), oder None."""
    if not os.path.exists(SWAP_BIN):
        return None
    try:
        out = subprocess.run([SWAP_BIN, "--version"], capture_output=True, text=True, timeout=15)
        txt = (out.stdout or "") + (out.stderr or "")
        if (m := re.search(r"version:\s*(\d+)", txt)) or (m := re.search(r"\bv?(\d{2,})\b", txt)):
            return int(m.group(1))
    except Exception:
        return None
    return None


def _swap_update_available() -> bool:
    now = time.time()
    if now - _swap_cache["ts"] < 3600:
        return _swap_cache["avail"]
    avail = False
    try:
        rel = httpx.get(f"https://api.github.com/repos/{SWAP_REPO}/releases/latest",
                        timeout=6, headers={"User-Agent": "MissionControl2"}).json()
        tag = str(rel.get("tag_name", ""))
        latest = int(m.group(1)) if (m := re.search(r"(\d{2,})", tag)) else None
        installed = _installed_swap_version()
        if latest is not None and installed is not None:
            avail = latest > installed
    except Exception:
        avail = False
    _swap_cache.update(ts=now, avail=avail)
    return avail


_comp_cache = {"ts": 0.0, "data": []}


def _hermes_agent_update() -> dict:
    """Hermes-Agent wird aus **git** aktualisiert (CLI `hermes update` = git pull origin <branch>).
    Darum HEAD vs. origin/<branch> prüfen (fetch + behind-count) — NICHT GitHub-Releases: die
    werden selten getaggt, main läuft ihnen voraus → sonst zeigt das UI nie ein Update an."""
    info = {"key": "hermes_agent", "name": "Hermes Agent", "current": None,
            "latest": None, "update": False, "reachable": None}
    git = system.find_hermes_agent_git()
    if not git or not git.get("path"):
        return info
    path = git["path"]
    info["current"] = git.get("hash")
    try:
        branch = (subprocess.run(["git", "-C", path, "rev-parse", "--abbrev-ref", "HEAD"],
                                 capture_output=True, text=True, timeout=8).stdout.strip() or "main")
        fetch = subprocess.run(["git", "-C", path, "fetch", "-q", "origin", branch],
                               capture_output=True, text=True, timeout=25)
        info["reachable"] = (fetch.returncode == 0)
        if fetch.returncode == 0:
            cnt = subprocess.run(["git", "-C", path, "rev-list", "--count", f"HEAD..origin/{branch}"],
                                 capture_output=True, text=True, timeout=8)
            behind = int(cnt.stdout.strip() or "0") if cnt.returncode == 0 else 0
            info["behind"] = behind
            info["update"] = behind > 0
            oh = subprocess.run(["git", "-C", path, "rev-parse", "--short", f"origin/{branch}"],
                                capture_output=True, text=True, timeout=8).stdout.strip()
            info["latest"] = (f"{oh} ({behind} neu)" if behind else (oh or info["current"]))
    except Exception:
        info["reachable"] = False
    return info


def _components_cached() -> list[dict]:
    """Update-Status von Hermes-Agent (1h-Cache → GitHub schonen)."""
    now = time.time()
    if now - _comp_cache["ts"] < 3600 and _comp_cache["data"]:
        return _comp_cache["data"]
    data = [_hermes_agent_update()]
    _comp_cache.update(ts=now, data=data)
    return data


def _params_of(m: dict) -> float:
    """Größen-bewusste Parameterzahl eines installierten Modells: max aus Namens-Schätzung
    und Dateigröße (fängt namenlose wie 'Qwen3-Coder-Next' UND Split-GGUFs ab)."""
    from services.fit import QUANT_BYTES_PER_PARAM
    caps = m.get("capabilities") or {}
    bpp = QUANT_BYTES_PER_PARAM.get((m.get("quant") or "Q4_K_M").upper(), 0.55)
    size_gb = (m.get("size_bytes") or 0) / (1024 ** 3)
    pb_size = (size_gb / bpp) if size_gb > 1.0 else 0.0
    return max(float(caps.get("params_b") or 0), pb_size, 0.0)


# Familien-Subtyp + Generations-Version aus dem Modellnamen (für „echtes Upgrade?").
_FAM_PATS = (("qwen", r"qwen(\d+(?:\.\d+)?)"), ("gemma", r"gemma[-_ ]?(\d+(?:\.\d+)?)"),
             ("llama", r"llama[-_ ]?(\d+(?:\.\d+)?)"), ("phi", r"phi[-_ ]?(\d+(?:\.\d+)?)"),
             ("mistral", r"mistral"), ("hermes", r"hermes[-_ ]?(\d+(?:\.\d+)?)"))


def _gen_key(name: str):
    """(Familie+Subtyp, Generations-Version) oder None. Z.B. 'Qwen3-VL-2B' → ('qwen-vl', 3.0),
    'Qwen2.5-VL-7B' → ('qwen-vl', 2.5). Nur gleiche Familie ist sinnvoll vergleichbar."""
    low = (name or "").lower()
    sub = "-vl" if any(k in low for k in ("-vl", "vl-", "vision", "llava", "pixtral")) else \
          "-coder" if ("coder" in low or "-code" in low) else ""
    for fam, pat in _FAM_PATS:
        m = re.search(pat, low)
        if m:
            ver = float(m.group(1)) if (m.groups() and m.group(1)) else 0.0
            return (fam + sub, ver)
    return None


def _meta(name: str, model_dict: dict | None = None, im: dict | None = None) -> dict:
    """Metadaten (family, gen, total, active, moe) — bevorzugt den kuratierten Katalog,
    sonst die Felder eines Discover-/Modell-Dicts, sonst Namens-/Größen-Heuristik."""
    cm = catalog.meta_for_name(name)
    if cm:
        return {"family": cm.get("family"), "gen": cm.get("generation"),
                "total": float(cm.get("total_params_b") or 0),
                "active": cm.get("active_params_b"), "moe": bool(cm.get("moe"))}
    d = model_dict or {}
    g = _gen_key(name)
    total = float(d.get("params_b") or 0) or (_params_of(im) if im else 0.0)
    return {"family": (d.get("family") or (g[0] if g else None)),
            "gen": (d.get("generation") if d.get("generation") is not None else (g[1] if g else None)),
            "total": total, "active": d.get("active_b"), "moe": bool(d.get("moe"))}


def model_upgrades() -> list[dict]:
    """Je Rolle ein ECHTES Upgrade — nur wenn die Empfehlung wirklich besser ist:
    gleiche Familie UND (neuere Generation ODER deutlich größer) UND kein Tempo-Downgrade
    (MoE-first für die bandbreiten-limitierte Box: dense ersetzt MoE nur bei großem Wissens-
    Sprung). Metadaten kommen aus dem kuratierten Katalog → keine Namens-Raterei."""
    disc = discover.safe_discover(_ram_gb())
    if not disc:
        return []

    installed = llamaswap.list_models()
    inst_by_role = {m["role"]: m for m in installed if m.get("role")}
    cmds = " ".join(str(s.get("cmd", "")).lower()
                    for s in (llamaswap.read_config().get("models") or {}).values())
    out = []

    for c in disc.get("categories", []):
        role = c["role"]
        im = inst_by_role.get(role)
        if im is None:
            continue
        rec = c.get("recommended")
        if not rec:
            continue
        rec_model = next((x for x in c.get("models", []) if x.get("repo") == rec), None)

        i = _meta(im["name"], im=im)
        r = _meta(rec, model_dict=rec_model)

        if not i["family"] or not r["family"] or i["family"] != r["family"]:
            continue                                   # andere/unbekannte Familie → kein Upgrade
        if r["gen"] is not None and i["gen"] is not None and r["gen"] < i["gen"] - 1e-6:
            continue                                   # ältere Generation → niemals
        same_gen = (r["gen"] is None or i["gen"] is None or abs(r["gen"] - i["gen"]) < 1e-6)
        if same_gen:
            if r["total"] and i["total"] and r["total"] < i["total"] * 1.05:
                continue                               # gleiche Gen, nicht größer → kein Upgrade
            # MoE-first: ein MoE durch dense ersetzen nur bei deutlichem Wissens-Sprung
            if i["moe"] and not r["moe"] and r["total"] < i["total"] * 1.5:
                continue
            # Tempo nicht verschlechtern (aktive Params), außer großer Wissens-Gewinn
            ia, ra = (i["active"] or i["total"]), (r["active"] or r["total"])
            if ia and ra > ia * 1.3 and r["total"] < i["total"] * 1.3:
                continue

        base = rec.split("/")[-1].lower()
        stem = base[:-5] if base.endswith("-gguf") else base
        if base in cmds or (stem and stem in cmds):
            continue                                   # schon installiert
        out.append({"role": role, "title": c["title"], "repo": rec})
    return out


def _last_apt_update() -> float | None:
    for path in ["/var/lib/apt/periodic/update-success-stamp", "/var/cache/apt/pkgcache.bin"]:
        if os.path.exists(path):
            try:
                return os.path.getmtime(path)
            except Exception:
                pass
    return None


def updates() -> dict:
    ups = model_upgrades()
    return {"os": _os_upgradable(), "engine": 1 if _engine_update_available() else 0,
            "swap": 1 if _swap_update_available() else 0,
            "models": len(ups), "model_list": ups, "last_check": _last_apt_update(),
            "components": _components_cached()}


# ── Update-Details (was genau wird aktualisiert) — on-demand beim Öffnen des Fensters ──

def os_update_details() -> dict:
    """Liste der aktualisierbaren apt-Pakete (Name, installiert → Kandidat)."""
    out_pkgs: list[dict] = []
    try:
        # LC_ALL=C → englische Ausgabe, damit der Regex "[upgradable from: ...]" greift
        # (deutsche Box meldet sonst "[aktualisierbar von:]" und die Liste bliebe leer).
        out = subprocess.run(["bash", "-c", "LC_ALL=C apt list --upgradable 2>/dev/null"],
                             capture_output=True, text=True, timeout=20)
        for line in (out.stdout or "").splitlines():
            # Format: name/repo  neue_version  arch  [upgradable from: alte_version]
            m = re.match(r"^([^/\s]+)/\S+\s+(\S+)\s+\S+\s+\[upgradable from:\s*([^\]]+)\]",
                         line.strip())
            if m:
                out_pkgs.append({"name": m.group(1), "candidate": m.group(2),
                                 "current": m.group(3).strip()})
        out_pkgs.sort(key=lambda p: p["name"])
        return {"kind": "os", "count": len(out_pkgs), "packages": out_pkgs}
    except Exception as exc:  # noqa: BLE001
        return {"kind": "os", "count": len(out_pkgs), "packages": out_pkgs, "error": str(exc)}


def engine_update_details() -> dict:
    """Installierte vs. neueste Engine-Build-Nummer + Release-Name/-Notizen/-Link."""
    info: dict = {"kind": "engine", "installed_build": _installed_engine_build(),
                  "latest_build": None, "latest_tag": None, "name": None,
                  "url": None, "body": None}
    try:
        rel = httpx.get(f"https://api.github.com/repos/{ENGINE_REPO}/releases/latest",
                        timeout=8, headers={"User-Agent": "MissionControl2"}).json()
        tag = str(rel.get("tag_name", ""))
        info["latest_tag"] = tag
        info["latest_build"] = int(m.group(1)) if (m := re.search(r"(\d{3,})", tag)) else None
        info["name"] = rel.get("name") or tag
        info["url"] = rel.get("html_url")
        body = (rel.get("body") or "").strip()
        info["body"] = body[:2000] if body else None
    except Exception as exc:  # noqa: BLE001
        info["error"] = str(exc)
    return info


def swap_update_details() -> dict:
    """Installierte vs. neueste llama-swap-Version + Release-Name/-Notizen/-Link."""
    info: dict = {"kind": "swap", "installed_build": _installed_swap_version(),
                  "latest_build": None, "latest_tag": None, "name": None,
                  "url": None, "body": None}
    try:
        rel = httpx.get(f"https://api.github.com/repos/{SWAP_REPO}/releases/latest",
                        timeout=8, headers={"User-Agent": "MissionControl2"}).json()
        tag = str(rel.get("tag_name", ""))
        info["latest_tag"] = tag
        info["latest_build"] = int(m.group(1)) if (m := re.search(r"(\d{2,})", tag)) else None
        info["name"] = rel.get("name") or tag
        info["url"] = rel.get("html_url")
        body = (rel.get("body") or "").strip()
        info["body"] = body[:2000] if body else None
    except Exception as exc:  # noqa: BLE001
        info["error"] = str(exc)
    return info


# LLM-Zusammenfassung der anstehenden Hermes-Commits (die Box liest ihre Release-Notes selbst).
# Gecacht auf den neuesten Commit-Hash — das Modal darf beliebig oft geöffnet werden.
_relnotes_cache: dict = {"key": "", "summary": ""}


def _summarize_hermes_commits(commits: list[dict]) -> str:
    key = commits[0]["hash"] if commits else ""
    if not key:
        return ""
    if _relnotes_cache["key"] == key and _relnotes_cache["summary"]:
        return _relnotes_cache["summary"]
    from config import LLAMA_SWAP_URL
    subjects = "\n".join(f"- {c['subject']}" for c in commits[:100])
    prompt = (
        "Du bist der Update-Berater einer lokalen AI-Box (Hermes-Agent auf Strix Halo; genutzt werden: "
        "api_server/Gateway, memory-provider-Plugin 'mc2-memory', terminal-/web-Tools, approvals, cron). "
        "Hier die Commit-Titel des anstehenden Hermes-Updates:\n\n" + subjects + "\n\n"
        "Fasse auf DEUTSCH in maximal 6 Stichpunkten zusammen: (1) Breaking Changes oder geänderte/"
        "entfernte Config-Schlüssel (WICHTIGSTES zuerst, explizit warnen), (2) was unser Setup betrifft, "
        "(3) welche neuen Features sich für uns lohnen. Keine Einleitung, nur die Punkte."
    )
    try:
        r = httpx.post(f"{LLAMA_SWAP_URL}/v1/chat/completions", timeout=90.0, json={
            "model": "fast", "max_tokens": 550, "temperature": 0.2,
            "chat_template_kwargs": {"enable_thinking": False},
            "messages": [{"role": "user", "content": prompt}],
        })
        r.raise_for_status()
        resp = r.json()
        choice = (resp.get("choices") or [{}])[0]
        summary = (choice.get("message") or {}).get("content") or ""
        finish_reason = choice.get("finish_reason") or ""
        summary = summary.strip()

        # finish_reason == "length" → Antwort wurde wegen Token-Limits abgeschnitten →
        # letzten (unvollständigen) Stichpunkt entfernen.
        # Bei "stop" oder None → Antwort ist vollständig → unverändert lassen.
        if finish_reason == "length" and summary:
            lines = summary.rsplit("\n", 1)
            summary = lines[0] if len(lines) > 1 else ""
        if summary:
            _relnotes_cache.update(key=key, summary=summary)
        return summary
    except Exception as exc:  # noqa: BLE001 — Zusammenfassung ist Komfort, nie Blocker
        return f"(Zusammenfassung nicht verfügbar: {exc})"


def hermes_update_details() -> dict:
    """Commits, die ein Hermes-Update einspielen würde (HEAD..origin/<branch>) + LLM-Zusammenfassung."""
    info: dict = {"kind": "hermes", "branch": None, "behind": 0, "commits": []}
    git = system.find_hermes_agent_git()
    if not git or not git.get("path"):
        info["error"] = "Hermes-Agent-Repo nicht gefunden."
        return info
    path = git["path"]
    try:
        branch = (subprocess.run(["git", "-C", path, "rev-parse", "--abbrev-ref", "HEAD"],
                                 capture_output=True, text=True, timeout=8).stdout.strip() or "main")
        info["branch"] = branch
        subprocess.run(["git", "-C", path, "fetch", "-q", "origin", branch],
                       capture_output=True, text=True, timeout=25)
        log = subprocess.run(["git", "-C", path, "log", "--pretty=format:%h\x1f%s\x1f%cr",
                              f"HEAD..origin/{branch}"], capture_output=True, text=True, timeout=10)
        commits = []
        for line in (log.stdout or "").splitlines():
            parts = line.split("\x1f")
            if len(parts) == 3:
                commits.append({"hash": parts[0], "subject": parts[1], "when": parts[2]})
        info["commits"] = commits
        info["behind"] = len(commits)
        if commits:
            info["summary"] = _summarize_hermes_commits(commits)
    except Exception as exc:  # noqa: BLE001
        info["error"] = str(exc)
    return info


def update_details(kind: str) -> dict:
    return {"os": os_update_details, "engine": engine_update_details,
            "swap": swap_update_details,
            "hermes": hermes_update_details}.get(kind, lambda: {"error": "unbekannt"})()


def _run(cmd: list[str], sudo_password: str | None = None) -> dict:
    actual_cmd = list(cmd)
    has_sudo = False
    
    if cmd and cmd[0] == "sudo":
        has_sudo = True
        # If we have a password, use -S instead of -n
        if sudo_password is not None:
            if "-n" in actual_cmd:
                actual_cmd = [x for x in actual_cmd if x != "-n"]
            if "-S" not in actual_cmd:
                actual_cmd.insert(1, "-S")
        else:
            # Force -n to fail cleanly if password is required
            if "-S" in actual_cmd:
                actual_cmd = [x for x in actual_cmd if x != "-S"]
            if "-n" not in actual_cmd:
                actual_cmd.insert(1, "-n")

    try:
        input_data = (sudo_password + "\n") if (has_sudo and sudo_password is not None) else None
        p = subprocess.run(actual_cmd, input=input_data, capture_output=True, text=True, timeout=120)
        
        err_msg = p.stderr or ""
        if p.returncode != 0 and ("a password is required" in err_msg or "password" in err_msg.lower() or "sudo:" in err_msg):
            if sudo_password is not None:
                return {"ok": False, "status": "incorrect_password", "out": p.stdout or "", "err": "Falsches Sudo-Passwort."}
            return {"ok": False, "status": "password_required", "out": p.stdout or "", "err": "Sudo-Passwort erforderlich."}
            
        return {"ok": p.returncode == 0, "out": (p.stdout or "")[-4000:], "err": (p.stderr or "")[-2000:]}
    except Exception as exc:  # noqa: BLE001
        return {"ok": False, "out": "", "err": str(exc)}


def check_sudo_needs_password(sudo_password: str | None = None) -> dict | None:
    """Checks if sudo needs a password. Returns error dict if password required/incorrect, else None."""
    res = _run(["sudo", "true"], sudo_password=sudo_password)
    if not res["ok"]:
        return res
    return None


def restart_service(name: str, sudo_password: str | None = None) -> dict:
    if name in SYSTEM_SERVICES:
        if err := check_sudo_needs_password(sudo_password):
            return err
        return _run(["sudo", "systemctl", "restart", name], sudo_password=sudo_password)
    if name in USER_SERVICES:
        return _run(["systemctl", "--user", "restart", name])
    return {"ok": False, "err": f"Dienst '{name}' nicht erlaubt."}


def logs(service: str, lines: int = 200, sudo_password: str | None = None) -> dict:
    lines = max(1, min(lines, 1000))
    if service in USER_SERVICES:
        r = _run(["journalctl", "--user", "-u", service, "-n", str(lines), "--no-pager"])
        return {"ok": r["ok"], "text": r["out"] or r["err"]}
    if service in SYSTEM_SERVICES:
        # Journal-Lesen braucht i.d.R. KEIN sudo (User ist in Gruppe adm/systemd-journal).
        # Erst ohne sudo versuchen; nur bei fehlenden Rechten auf sudo zurückfallen.
        r = _run(["journalctl", "-u", service, "-n", str(lines), "--no-pager"])
        if r["ok"]:
            return {"ok": True, "text": r["out"] or "(keine Log-Einträge)"}
        if err := check_sudo_needs_password(sudo_password):
            return err
        r = _run(["sudo", "journalctl", "-u", service, "-n", str(lines), "--no-pager"],
                 sudo_password=sudo_password)
        return {"ok": r["ok"], "text": r["out"] or r["err"]}
    return {"ok": False, "text": "", "err": "Dienst nicht erlaubt."}

def check_updates_job(sudo_password: str | None = None) -> dict:
    if err := check_sudo_needs_password(sudo_password):
        return err
    
    def on_done():
        _engine_cache.update(ts=0.0, avail=False)
        _comp_cache.update(ts=0.0, data=[])  # Hermes-Status ebenfalls neu berechnen lassen

    cmd = "sudo apt-get update"
    job_id = jobengine.start_job(["bash", "-c", cmd], "Nach Updates suchen", on_done=on_done, sudo_password=sudo_password)
    return {"ok": True, "job_id": job_id}


def _maintenance_busy() -> dict | None:
    """Wartungs-Riegel: nur EIN binär-/dienst-veränderndes Update gleichzeitig. Verhindert
    Doppelklick UND parallele Updates aus zwei Tabs/Sessions (racende .bak-Sicherung/Restarts)."""
    if j := jobengine.active_in_group("maintenance"):
        return {"ok": False, "status": "busy", "running": j.get("label")}
    return None


def os_update_job(sudo_password: str | None = None) -> dict:
    if busy := _maintenance_busy():
        return busy
    if err := check_sudo_needs_password(sudo_password):
        return err
    # Nach dem apt-Upgrade den Stack funktional prüfen (Job wird rot, wenn etwas kaputt ging).
    cmd = ("sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y "
           f"&& bash {STACK_POSTCHECK}")
    job_id = jobengine.start_job(["bash", "-c", cmd], "OS-Update (apt)",
                                 group="maintenance", sudo_password=sudo_password)
    return {"ok": True, "job_id": job_id}


def engine_update_job(sudo_password: str | None = None) -> dict | None:
    if not ENGINE_UPDATE_CMD:
        return None
    if busy := _maintenance_busy():
        return busy
    if err := check_sudo_needs_password(sudo_password):
        return err

    def on_done():
        _engine_cache.update(ts=0.0, avail=False)  # Cache leeren → frischer Build-Vergleich

    # update-engine.sh sichert den alten Build, aktualisiert, startet llama-swap neu, prüft den
    # Stack (stack-postcheck.sh) und rollt bei Fehler selbst zurück. Exit 0 nur bei verifiziertem
    # neuen Build → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
    job_id = jobengine.start_job(["bash", "-c", ENGINE_UPDATE_CMD],
                                 "Engine-Update (llama.cpp Vulkan)",
                                 group="maintenance", on_done=on_done, sudo_password=sudo_password)
    return {"ok": True, "job_id": job_id}


def swap_update_job(sudo_password: str | None = None) -> dict | None:
    if not SWAP_UPDATE_CMD:
        return None
    if busy := _maintenance_busy():
        return busy
    if err := check_sudo_needs_password(sudo_password):
        return err

    def on_done():
        _swap_cache.update(ts=0.0, avail=False)  # Cache leeren → frischer Versions-Vergleich

    # update-swap.sh sichert die alte Binary, aktualisiert, startet llama-swap neu, prüft den Stack
    # (stack-postcheck.sh) und rollt bei Fehler selbst zurück. Exit 0 nur bei verifizierter neuer
    # Version → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
    job_id = jobengine.start_job(["bash", "-c", SWAP_UPDATE_CMD],
                                 "Router-Update (llama-swap)",
                                 group="maintenance", on_done=on_done, sudo_password=sudo_password)
    return {"ok": True, "job_id": job_id}


def hermes_update_job() -> dict:
    """Hermes-Agent aktualisieren wie die CLI (`hermes update` = git pull + Deps), danach
    den Gateway neu starten. Davor ein Sicherheits-Backup (unser deploy/backup.sh). Kein sudo
    (alles im User-Space). Läuft als Hintergrund-Job (kann ~1 Min dauern)."""
    if busy := _maintenance_busy():
        return busy
    git = system.find_hermes_agent_git()
    path = (git or {}).get("path") or os.path.expanduser("~/.hermes/hermes-agent")
    py = os.path.join(path, "venv", "bin", "python")
    backup = os.path.join(_REPO_ROOT, "deploy", "backup.sh")
    postcheck = os.path.join(_REPO_ROOT, "deploy", "hermes-postcheck.sh")
    # Backup → update → Gateway-Neustart → Gehirn-Check (Job wird rot, wenn Mem0/Plugin kaputt).
    # Backup → update → doctor (Config/Deps-Diagnose der NEUEN Version) → Gateway-Neustart →
    # Gehirn-Check (erweitert um Config-Drift-Scan, Tool- und Voice-Smoke — Lehre aus v0.18:
    # 'approvals.mode auto' wurde still ungültig und legte alle Tools in pending_approval).
    cmd = (f"bash {backup} || true; "
           f"cd {path} && {py} -m hermes_cli.main update --yes "
           f"&& {py} -m hermes_cli.main doctor "
           f"&& systemctl --user restart hermes-gateway "
           f"&& sleep 6 && bash {postcheck}")

    def on_done():
        _comp_cache.update(ts=0.0, data=[])  # Update-Status neu berechnen lassen

    job_id = jobengine.start_job(["bash", "-c", cmd], "Hermes-Agent-Update",
                                 group="maintenance", on_done=on_done)
    return {"ok": True, "job_id": job_id}


def reboot(sudo_password: str | None = None) -> dict:
    if err := check_sudo_needs_password(sudo_password):
        return err
    return _run(["sudo", "reboot"], sudo_password=sudo_password)