Feat: Wartungs-Riegel — nur EIN System-Update gleichzeitig
Bisher kein Schutz: Doppelklick/zwei Tabs konnten zwei update-engine.sh parallel
starten → racende .bak-Sicherung + parallele llama-swap-Restarts + sich gegenseitig
als kaputt sehende Postchecks.
Backend: jobengine.start_job bekommt group-Tag + active_in_group(); os/engine/swap/
hermes-update sind group="maintenance" und lehnen einen Start ab, solange eines laeuft
({ok:false, status:"busy", running:<label>}). Schuetzt auch gegen parallele Sessions.
Frontend: laeuft ein Wartungs-Job, zeigt der Drawer ein Banner "Update laeuft: <label>"
und sperrt "Jetzt aktualisieren" + "Nach Updates suchen". Logs/Job-Fortschritt/Dienste
bleiben voll nutzbar (Dashboard nicht hart gesperrt). Busy-Antwort wird als Hinweis gezeigt.
Modell-Upgrades bleiben erlaubt (parallel unkritisch).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -147,12 +147,13 @@ def attach_download_progress(job_id: str, local_dir: str, total_bytes: int) -> N
|
|||||||
threading.Thread(target=_watch, daemon=True).start()
|
threading.Thread(target=_watch, daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
def start_job(args: list[str], label: str, env: dict | None = None, on_done=None, sudo_password: str | None = None) -> str:
|
def start_job(args: list[str], label: str, env: dict | None = None, on_done=None,
|
||||||
|
sudo_password: str | None = None, group: str | None = None) -> str:
|
||||||
job_id = uuid.uuid4().hex[:12]
|
job_id = uuid.uuid4().hex[:12]
|
||||||
# Mask password in log if present in args
|
# Mask password in log if present in args
|
||||||
log_args = list(args)
|
log_args = list(args)
|
||||||
JOBS[job_id] = {
|
JOBS[job_id] = {
|
||||||
"id": job_id, "label": label, "state": "queued",
|
"id": job_id, "label": label, "state": "queued", "group": group,
|
||||||
"log": ["$ " + " ".join(shlex.quote(a) for a in log_args)],
|
"log": ["$ " + " ".join(shlex.quote(a) for a in log_args)],
|
||||||
"returncode": None, "started_at": time.time(), "finished_at": None,
|
"returncode": None, "started_at": time.time(), "finished_at": None,
|
||||||
}
|
}
|
||||||
@@ -162,6 +163,15 @@ def start_job(args: list[str], label: str, env: dict | None = None, on_done=None
|
|||||||
return job_id
|
return job_id
|
||||||
|
|
||||||
|
|
||||||
|
def active_in_group(group: str) -> dict | None:
|
||||||
|
"""Erster laufender/wartender Job einer Gruppe (z.B. 'maintenance'), sonst None.
|
||||||
|
Basis für den Wartungs-Riegel: nur EIN System-Update gleichzeitig."""
|
||||||
|
for j in JOBS.values():
|
||||||
|
if j.get("group") == group and j.get("state") in ("running", "queued"):
|
||||||
|
return j
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def cancel_job(job_id: str) -> bool:
|
def cancel_job(job_id: str) -> bool:
|
||||||
job = JOBS.get(job_id)
|
job = JOBS.get(job_id)
|
||||||
if not job or job["state"] in ("done", "failed", "canceled"):
|
if not job or job["state"] in ("done", "failed", "canceled"):
|
||||||
|
|||||||
@@ -476,19 +476,32 @@ def check_updates_job(sudo_password: str | None = None) -> dict:
|
|||||||
return {"ok": True, "job_id": job_id}
|
return {"ok": True, "job_id": job_id}
|
||||||
|
|
||||||
|
|
||||||
|
def _maintenance_busy() -> dict | None:
|
||||||
|
"""Wartungs-Riegel: nur EIN binär-/dienst-veränderndes Update gleichzeitig. Verhindert
|
||||||
|
Doppelklick UND parallele Updates aus zwei Tabs/Sessions (racende .bak-Sicherung/Restarts)."""
|
||||||
|
if j := jobengine.active_in_group("maintenance"):
|
||||||
|
return {"ok": False, "status": "busy", "running": j.get("label")}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def os_update_job(sudo_password: str | None = None) -> dict:
|
def os_update_job(sudo_password: str | None = None) -> dict:
|
||||||
|
if busy := _maintenance_busy():
|
||||||
|
return busy
|
||||||
if err := check_sudo_needs_password(sudo_password):
|
if err := check_sudo_needs_password(sudo_password):
|
||||||
return err
|
return err
|
||||||
# Nach dem apt-Upgrade den Stack funktional prüfen (Job wird rot, wenn etwas kaputt ging).
|
# Nach dem apt-Upgrade den Stack funktional prüfen (Job wird rot, wenn etwas kaputt ging).
|
||||||
cmd = ("sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y "
|
cmd = ("sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y "
|
||||||
f"&& bash {STACK_POSTCHECK}")
|
f"&& bash {STACK_POSTCHECK}")
|
||||||
job_id = jobengine.start_job(["bash", "-c", cmd], "OS-Update (apt)", sudo_password=sudo_password)
|
job_id = jobengine.start_job(["bash", "-c", cmd], "OS-Update (apt)",
|
||||||
|
group="maintenance", sudo_password=sudo_password)
|
||||||
return {"ok": True, "job_id": job_id}
|
return {"ok": True, "job_id": job_id}
|
||||||
|
|
||||||
|
|
||||||
def engine_update_job(sudo_password: str | None = None) -> dict | None:
|
def engine_update_job(sudo_password: str | None = None) -> dict | None:
|
||||||
if not ENGINE_UPDATE_CMD:
|
if not ENGINE_UPDATE_CMD:
|
||||||
return None
|
return None
|
||||||
|
if busy := _maintenance_busy():
|
||||||
|
return busy
|
||||||
if err := check_sudo_needs_password(sudo_password):
|
if err := check_sudo_needs_password(sudo_password):
|
||||||
return err
|
return err
|
||||||
|
|
||||||
@@ -500,13 +513,15 @@ def engine_update_job(sudo_password: str | None = None) -> dict | None:
|
|||||||
# neuen Build → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
|
# neuen Build → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
|
||||||
job_id = jobengine.start_job(["bash", "-c", ENGINE_UPDATE_CMD],
|
job_id = jobengine.start_job(["bash", "-c", ENGINE_UPDATE_CMD],
|
||||||
"Engine-Update (llama.cpp Vulkan)",
|
"Engine-Update (llama.cpp Vulkan)",
|
||||||
on_done=on_done, sudo_password=sudo_password)
|
group="maintenance", on_done=on_done, sudo_password=sudo_password)
|
||||||
return {"ok": True, "job_id": job_id}
|
return {"ok": True, "job_id": job_id}
|
||||||
|
|
||||||
|
|
||||||
def swap_update_job(sudo_password: str | None = None) -> dict | None:
|
def swap_update_job(sudo_password: str | None = None) -> dict | None:
|
||||||
if not SWAP_UPDATE_CMD:
|
if not SWAP_UPDATE_CMD:
|
||||||
return None
|
return None
|
||||||
|
if busy := _maintenance_busy():
|
||||||
|
return busy
|
||||||
if err := check_sudo_needs_password(sudo_password):
|
if err := check_sudo_needs_password(sudo_password):
|
||||||
return err
|
return err
|
||||||
|
|
||||||
@@ -518,7 +533,7 @@ def swap_update_job(sudo_password: str | None = None) -> dict | None:
|
|||||||
# Version → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
|
# Version → on_done (Cache leeren) läuft nur dann; bei Rollback (Exit 1/2) bleibt das Badge.
|
||||||
job_id = jobengine.start_job(["bash", "-c", SWAP_UPDATE_CMD],
|
job_id = jobengine.start_job(["bash", "-c", SWAP_UPDATE_CMD],
|
||||||
"Router-Update (llama-swap)",
|
"Router-Update (llama-swap)",
|
||||||
on_done=on_done, sudo_password=sudo_password)
|
group="maintenance", on_done=on_done, sudo_password=sudo_password)
|
||||||
return {"ok": True, "job_id": job_id}
|
return {"ok": True, "job_id": job_id}
|
||||||
|
|
||||||
|
|
||||||
@@ -526,6 +541,8 @@ def hermes_update_job() -> dict:
|
|||||||
"""Hermes-Agent aktualisieren wie die CLI (`hermes update` = git pull + Deps), danach
|
"""Hermes-Agent aktualisieren wie die CLI (`hermes update` = git pull + Deps), danach
|
||||||
den Gateway neu starten. Davor ein Sicherheits-Backup (unser deploy/backup.sh). Kein sudo
|
den Gateway neu starten. Davor ein Sicherheits-Backup (unser deploy/backup.sh). Kein sudo
|
||||||
(alles im User-Space). Läuft als Hintergrund-Job (kann ~1 Min dauern)."""
|
(alles im User-Space). Läuft als Hintergrund-Job (kann ~1 Min dauern)."""
|
||||||
|
if busy := _maintenance_busy():
|
||||||
|
return busy
|
||||||
git = system.find_hermes_agent_git()
|
git = system.find_hermes_agent_git()
|
||||||
path = (git or {}).get("path") or os.path.expanduser("~/.hermes/hermes-agent")
|
path = (git or {}).get("path") or os.path.expanduser("~/.hermes/hermes-agent")
|
||||||
py = os.path.join(path, "venv", "bin", "python")
|
py = os.path.join(path, "venv", "bin", "python")
|
||||||
@@ -540,7 +557,8 @@ def hermes_update_job() -> dict:
|
|||||||
def on_done():
|
def on_done():
|
||||||
_comp_cache.update(ts=0.0, data=[]) # Update-Status neu berechnen lassen
|
_comp_cache.update(ts=0.0, data=[]) # Update-Status neu berechnen lassen
|
||||||
|
|
||||||
job_id = jobengine.start_job(["bash", "-c", cmd], "Hermes-Agent-Update", on_done=on_done)
|
job_id = jobengine.start_job(["bash", "-c", cmd], "Hermes-Agent-Update",
|
||||||
|
group="maintenance", on_done=on_done)
|
||||||
return {"ok": True, "job_id": job_id}
|
return {"ok": True, "job_id": job_id}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
File diff suppressed because one or more lines are too long
+143
-143
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
|||||||
<link rel="manifest" href="/manifest.webmanifest" />
|
<link rel="manifest" href="/manifest.webmanifest" />
|
||||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||||
<title>Mission Control 2.0</title>
|
<title>Mission Control 2.0</title>
|
||||||
<script type="module" crossorigin src="/assets/index-CWx_Bgrw.js"></script>
|
<script type="module" crossorigin src="/assets/index-4xoYNTUx.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-BFWF7uFR.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-BFWF7uFR.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -217,9 +217,20 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
loadServiceLogs(selectedService)
|
loadServiceLogs(selectedService)
|
||||||
}, [open, activeTab, selectedService])
|
}, [open, activeTab, selectedService])
|
||||||
|
|
||||||
|
// Backend-Wartungsriegel: lehnt ein zweites Update ab, solange eines läuft.
|
||||||
|
function handledBusy(res: any): boolean {
|
||||||
|
if (res?.status === "busy") {
|
||||||
|
showAlert("Update läuft bereits", `Es läuft gerade „${res.running}". Bitte warte, bis es fertig ist.`)
|
||||||
|
loadJobs()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
async function triggerOsUpdate() {
|
async function triggerOsUpdate() {
|
||||||
try {
|
try {
|
||||||
await api<{ job_id: string }>("/api/maintenance/os-update", { method: "POST" })
|
const res = await api<any>("/api/maintenance/os-update", { method: "POST" })
|
||||||
|
if (handledBusy(res)) return
|
||||||
loadJobs()
|
loadJobs()
|
||||||
setActiveTab("maintenance")
|
setActiveTab("maintenance")
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
@@ -229,7 +240,8 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
|
|
||||||
async function triggerEngineUpdate() {
|
async function triggerEngineUpdate() {
|
||||||
try {
|
try {
|
||||||
await api<{ job_id: string }>("/api/maintenance/engine-update", { method: "POST" })
|
const res = await api<any>("/api/maintenance/engine-update", { method: "POST" })
|
||||||
|
if (handledBusy(res)) return
|
||||||
loadJobs()
|
loadJobs()
|
||||||
setActiveTab("maintenance")
|
setActiveTab("maintenance")
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
@@ -239,7 +251,8 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
|
|
||||||
async function triggerSwapUpdate() {
|
async function triggerSwapUpdate() {
|
||||||
try {
|
try {
|
||||||
await api<{ job_id: string }>("/api/maintenance/swap-update", { method: "POST" })
|
const res = await api<any>("/api/maintenance/swap-update", { method: "POST" })
|
||||||
|
if (handledBusy(res)) return
|
||||||
loadJobs()
|
loadJobs()
|
||||||
setActiveTab("maintenance")
|
setActiveTab("maintenance")
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
@@ -250,7 +263,8 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
async function doHermesUpdate() {
|
async function doHermesUpdate() {
|
||||||
setHermesUpdating(true)
|
setHermesUpdating(true)
|
||||||
try {
|
try {
|
||||||
await api<{ job_id: string }>("/api/maintenance/hermes-update", { method: "POST" })
|
const res = await api<any>("/api/maintenance/hermes-update", { method: "POST" })
|
||||||
|
if (handledBusy(res)) return
|
||||||
loadJobs()
|
loadJobs()
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
showAlert("Fehler", `Hermes-Update fehlgeschlagen: ${e.message}`)
|
showAlert("Fehler", `Hermes-Update fehlgeschlagen: ${e.message}`)
|
||||||
@@ -370,6 +384,9 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Aktiver Wartungs-Job (System-Update) → UI-Aktionen sperren, Dashboard bleibt sichtbar.
|
||||||
|
const maintenanceJob = jobs.find(j => (j.state === "running" || j.state === "queued") && j.group === "maintenance")
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
{/* Backdrop */}
|
{/* Backdrop */}
|
||||||
@@ -447,7 +464,7 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
<div className="space-y-2.5">
|
<div className="space-y-2.5">
|
||||||
<div className="flex items-center justify-between">
|
<div className="flex items-center justify-between">
|
||||||
<h3 className="text-xs font-bold uppercase tracking-wider text-muted-foreground">Updates</h3>
|
<h3 className="text-xs font-bold uppercase tracking-wider text-muted-foreground">Updates</h3>
|
||||||
<button onClick={triggerCheckUpdates} disabled={checkingUpdates}
|
<button onClick={triggerCheckUpdates} disabled={checkingUpdates || !!maintenanceJob}
|
||||||
className="flex items-center gap-1 text-[10px] font-semibold text-primary hover:text-primary/80 transition-colors disabled:opacity-50">
|
className="flex items-center gap-1 text-[10px] font-semibold text-primary hover:text-primary/80 transition-colors disabled:opacity-50">
|
||||||
<RefreshCw className={cn("h-3 w-3", checkingUpdates && "animate-spin")} /> Nach Updates suchen
|
<RefreshCw className={cn("h-3 w-3", checkingUpdates && "animate-spin")} /> Nach Updates suchen
|
||||||
</button>
|
</button>
|
||||||
@@ -455,6 +472,12 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
{updates?.last_check && (
|
{updates?.last_check && (
|
||||||
<div className="text-[9px] text-muted-foreground -mt-1">Zuletzt gesucht: {formatLastCheck(updates.last_check)}</div>
|
<div className="text-[9px] text-muted-foreground -mt-1">Zuletzt gesucht: {formatLastCheck(updates.last_check)}</div>
|
||||||
)}
|
)}
|
||||||
|
{maintenanceJob && (
|
||||||
|
<div className="flex items-center gap-2 rounded-lg border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-[11px] text-amber-300">
|
||||||
|
<RefreshCw className="h-3.5 w-3.5 shrink-0 animate-spin" />
|
||||||
|
<span>Update läuft: <span className="font-semibold">{maintenanceJob.label}</span> — bitte warten. Weitere Updates sind solange gesperrt.</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
<div className="space-y-1.5">
|
<div className="space-y-1.5">
|
||||||
<UpdateRow icon={Shield} iconClass="text-cyan-400" name="OS-Pakete (apt)" available={!!updates?.os} status={updates?.os ? `${updates.os} verfügbar` : "aktuell"} actionLabel="Anzeigen" onAction={() => openUpdateDetails("os")} />
|
<UpdateRow icon={Shield} iconClass="text-cyan-400" name="OS-Pakete (apt)" available={!!updates?.os} status={updates?.os ? `${updates.os} verfügbar` : "aktuell"} actionLabel="Anzeigen" onAction={() => openUpdateDetails("os")} />
|
||||||
<UpdateRow icon={Server} iconClass="text-violet-400" name="Inferenz-Engine (llama.cpp)" available={!!updates?.engine} status={updates?.engine ? "Update verfügbar" : "aktuell"} actionLabel="Anzeigen" onAction={() => openUpdateDetails("engine")} />
|
<UpdateRow icon={Server} iconClass="text-violet-400" name="Inferenz-Engine (llama.cpp)" available={!!updates?.engine} status={updates?.engine ? "Update verfügbar" : "aktuell"} actionLabel="Anzeigen" onAction={() => openUpdateDetails("engine")} />
|
||||||
@@ -876,9 +899,10 @@ export function SystemDrawer({ open, onClose, defaultTab = "maintenance" }: Syst
|
|||||||
<button onClick={() => setDetail(null)} className="h-9 flex-1 rounded-lg border border-border/60 bg-background/20 text-xs font-semibold text-muted-foreground hover:bg-accent transition-colors cursor-pointer">
|
<button onClick={() => setDetail(null)} className="h-9 flex-1 rounded-lg border border-border/60 bg-background/20 text-xs font-semibold text-muted-foreground hover:bg-accent transition-colors cursor-pointer">
|
||||||
Schließen
|
Schließen
|
||||||
</button>
|
</button>
|
||||||
<button onClick={applyFromDetail} disabled={detail.loading || nothing}
|
<button onClick={applyFromDetail} disabled={detail.loading || nothing || !!maintenanceJob}
|
||||||
|
title={maintenanceJob ? `Update läuft bereits: ${maintenanceJob.label}` : undefined}
|
||||||
className="h-9 flex-1 rounded-lg bg-primary text-xs font-semibold text-primary-foreground hover:opacity-90 transition-all cursor-pointer disabled:opacity-40 disabled:cursor-default">
|
className="h-9 flex-1 rounded-lg bg-primary text-xs font-semibold text-primary-foreground hover:opacity-90 transition-all cursor-pointer disabled:opacity-40 disabled:cursor-default">
|
||||||
Jetzt aktualisieren
|
{maintenanceJob ? "Update läuft…" : "Jetzt aktualisieren"}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -337,6 +337,7 @@ export interface AgentStatus {
|
|||||||
export interface Job {
|
export interface Job {
|
||||||
id: string
|
id: string
|
||||||
label: string
|
label: string
|
||||||
|
group?: string | null // "maintenance" = system-veränderndes Update (Wartungs-Riegel)
|
||||||
state: "queued" | "running" | "done" | "failed" | "canceled"
|
state: "queued" | "running" | "done" | "failed" | "canceled"
|
||||||
progress?: number
|
progress?: number
|
||||||
total_bytes?: number
|
total_bytes?: number
|
||||||
|
|||||||
Reference in New Issue
Block a user