feat: lower memory dedupe threshold for more aggressive cleaning
This commit is contained in:
root
+264
-264
@@ -1,264 +1,264 @@
|
||||
"""
|
||||
Hermes-Agent-Status (Control-Plane-Read). MC betreibt Hermes NICHT — es zeigt nur
|
||||
Status + verlinkt das standalone hermes-webui. Voller Zugriff + Tools/MCP werden in
|
||||
Hermes' eigener Config verdrahtet (siehe docs/HERMES_SETUP.md).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
|
||||
from config import (HERMES_TERMINAL_UPSTREAM, HERMES_TERMINAL_PATH, BOX_CONSOLE_UPSTREAM,
|
||||
BOX_CONSOLE_PATH, HERMES_BUILTIN_UI_UPSTREAM, HERMES_BUILTIN_UI_PATH,
|
||||
HERMES_API_URL, HERMES_HOME, PC_EXECUTOR_URL)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _hermes_version(name: str) -> float | None:
|
||||
"""Versionszahl aus 'Hermes-4.3', 'Hermes-4', 'Nous-Hermes-2' → 4.3/4.0/2.0."""
|
||||
low = (name or "").lower()
|
||||
if "hermes" not in low:
|
||||
return None
|
||||
m = re.search(r"hermes[-_ ]?(\d+(?:\.\d+)?)", low)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _active_brain_name() -> str:
|
||||
"""Aktives Agent-Hirn aus Hermes' Config: model.default (sonst model.model)."""
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
p = HERMES_HOME / "config.yaml"
|
||||
if p.exists():
|
||||
with p.open(encoding="utf-8") as f:
|
||||
cfg = YAML().load(f) or {}
|
||||
m = (cfg.get("model") or {}) if isinstance(cfg, dict) else {}
|
||||
return str(m.get("default") or m.get("model") or "auto")
|
||||
except Exception:
|
||||
log.debug("_active_brain_name: Lesefehler", exc_info=True)
|
||||
return "auto"
|
||||
|
||||
|
||||
def hermes_brain_info() -> dict:
|
||||
"""Aktuelles Agent-Hirn = Modell/Alias, das Hermes laut Config nutzt (model.default),
|
||||
plus Budget-Check. Zeigt das REAL genutzte Hirn — unabhängig von einer 'hermes'-Rolle."""
|
||||
from services import llamaswap
|
||||
|
||||
models = llamaswap.list_models()
|
||||
brain = _active_brain_name() # z.B. "fast" (Alias) oder ein Modellname
|
||||
bl = brain.lower()
|
||||
cur = next((m for m in models if (m.get("role") or "").lower() == bl), None) \
|
||||
or next((m for m in models if bl in (m["name"] or "").lower()), None)
|
||||
cur_params = (cur.get("capabilities") or {}).get("params_b") if cur else None
|
||||
current = None
|
||||
if cur:
|
||||
current = {"name": cur["name"], "alias": brain, "filename": cur.get("filename"),
|
||||
"params_b": cur_params, "quant": cur.get("quant"),
|
||||
"size_bytes": cur.get("size_bytes"),
|
||||
"gguf_path": cur.get("gguf_path"), "incomplete": cur.get("incomplete")}
|
||||
|
||||
# Fit-Check: passt das (immer warme) Hirn + das größte on-demand-Modell zusammen ins Budget?
|
||||
budget = None
|
||||
try:
|
||||
from services.budget import footprint_gb, gtt_budget_gb
|
||||
groups = llamaswap.list_groups()
|
||||
persist = set()
|
||||
for g in groups.values():
|
||||
if isinstance(g, dict) and (g.get("persist") or g.get("persistent")):
|
||||
persist.update(g.get("members") or [])
|
||||
|
||||
cur_name = cur["name"] if cur else None
|
||||
brain_gb = footprint_gb(cur) if cur else 0.0
|
||||
# voller Always-Warm-Footprint (alle persist, Brain=Empfehlung) — nur Info
|
||||
warm = brain_gb + sum(footprint_gb(m) for m in models
|
||||
if m["name"] in persist and m["name"] != cur_name)
|
||||
largest_od = max((footprint_gb(m) for m in models if m["name"] not in persist), default=0.0)
|
||||
gtt = gtt_budget_gb()
|
||||
# Seit dem `persistent`-Fix (03.07.) bleibt das GANZE Warmset (Hirn+embed+vision)
|
||||
# resident, wenn ein on-demand-Modell DANEBEN lädt → der reale Peak ist Warmset +
|
||||
# größtes on-demand, nicht nur Hirn + größtes. Genau daran wird `fits` gemessen.
|
||||
budget = {
|
||||
"gtt_gb": gtt,
|
||||
"brain_gb": round(brain_gb, 1),
|
||||
"warm_projected_gb": round(warm, 1),
|
||||
"largest_ondemand_gb": round(largest_od, 1),
|
||||
"fits": (warm + largest_od) <= gtt,
|
||||
"free_after_gb": round(gtt - warm - largest_od, 1),
|
||||
}
|
||||
except Exception:
|
||||
log.debug("hermes_brain_info: Budget-Berechnung fehlgeschlagen", exc_info=True)
|
||||
|
||||
return {"current": current, "recommended": None, "update_available": False, "budget": budget}
|
||||
|
||||
|
||||
def _reach(url: str, path: str = "") -> bool:
|
||||
try:
|
||||
with httpx.Client(timeout=3.0) as c:
|
||||
return c.get(f"{url}{path}").status_code < 500
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
def _count_enabled_mcp_servers() -> int:
|
||||
config_path = HERMES_HOME / "config.yaml"
|
||||
if not config_path.exists():
|
||||
return 0
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
mcp_servers = cfg.get("mcp_servers", {}) if isinstance(cfg, dict) else {}
|
||||
if not isinstance(mcp_servers, dict):
|
||||
return 0
|
||||
return sum(1 for v in mcp_servers.values() if isinstance(v, dict) and v.get("enabled", True))
|
||||
except Exception:
|
||||
log.debug("_count_enabled_mcp_servers: Fehler", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def agent_status() -> dict:
|
||||
"""Erreichbarkeit von Gateway (:8642) + WebUI (:8787) + lokale Hinweise."""
|
||||
home = HERMES_HOME
|
||||
brain_model = "auto"
|
||||
config_path = home / "config.yaml"
|
||||
if config_path.exists():
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
if isinstance(cfg, dict):
|
||||
# Hermes nutzt model.default als aktives Modell (model.model = Provider-Param).
|
||||
m = cfg.get("model", {}) or {}
|
||||
brain_model = m.get("default") or m.get("model") or "auto"
|
||||
except Exception:
|
||||
log.debug("agent_status: Hermes-config.yaml nicht lesbar", exc_info=True)
|
||||
|
||||
|
||||
return {
|
||||
"gateway_url": HERMES_API_URL,
|
||||
# Interaktives Web-Terminal (ttyd → `hermes chat`): same-origin über MC2 geproxyt.
|
||||
"terminal_url": HERMES_TERMINAL_PATH,
|
||||
# Box-Konsole (ttyd → Login-Shell): same-origin über MC2 geproxyt (/console/).
|
||||
"box_console_url": BOX_CONSOLE_PATH,
|
||||
# Eingebaute Hermes-Web-GUI (hermes serve): same-origin über MC2 geproxyt (/hermes-ui/).
|
||||
"hermes_ui_url": HERMES_BUILTIN_UI_PATH,
|
||||
"gateway_reachable": _reach(HERMES_API_URL, "/health"),
|
||||
# Erreichbarkeit der lokalen ttyd-Upstreams (Loopback, je base-path).
|
||||
"terminal_reachable": _reach(HERMES_TERMINAL_UPSTREAM, "/hermes-terminal/"),
|
||||
"box_console_reachable": _reach(BOX_CONSOLE_UPSTREAM, "/console/"),
|
||||
# Erreichbarkeit der eingebauten Hermes-GUI (Loopback :9119).
|
||||
"hermes_ui_reachable": _reach(HERMES_BUILTIN_UI_UPSTREAM, "/"),
|
||||
"home_exists": home.exists(),
|
||||
"brain_model": brain_model,
|
||||
# Best-effort: welche Verdrahtung lokal sichtbar ist (auf der Box aussagekräftig).
|
||||
"has_config": (home / "config.yaml").exists() or (home / "config.json").exists(),
|
||||
"has_skills": (home / "skills").exists(),
|
||||
"has_memories": (home / "memories").exists(),
|
||||
# Neue Felder: Telegram, MCP-Server-Anzahl, PC-Executor-Erreichbarkeit.
|
||||
"telegram_enabled": bool(os.environ.get("TELEGRAM_BOT_TOKEN", "")),
|
||||
"mcp_server_count": _count_enabled_mcp_servers(),
|
||||
"pc_executor_reachable": _reach(PC_EXECUTOR_URL, "/health"),
|
||||
}
|
||||
|
||||
|
||||
def set_agent_brain(model_id: str) -> dict:
|
||||
"""Setzt ein (bereits installiertes) Modell als Agent-Hirn — WARM-bewusst:
|
||||
1) vergibt den 'hermes'-Alias (das Agent-Hirn-Slot),
|
||||
2) tauscht es in die residente brains-Gruppe (altes Hirn raus, fast/vision bleiben),
|
||||
3) zeigt die Hermes-Config auf den 'hermes'-Alias + Gateway-Restart.
|
||||
So bleibt das neue Hirn warm und der Agent nutzt es sofort."""
|
||||
from services import llamaswap
|
||||
models = {m["name"]: m for m in llamaswap.list_models()}
|
||||
if model_id not in models:
|
||||
return {"ok": False, "reason": "Modell nicht installiert — erst über Modelle-finden laden."}
|
||||
old = next((m["name"] for m in models.values() if m.get("role") == "hermes"), None)
|
||||
if model_id == old:
|
||||
# Idempotent härten: auch wenn schon Hirn, warm (brains) + ttl 0 sicherstellen.
|
||||
try:
|
||||
from services.llamaswap import set_ttl
|
||||
brains = (llamaswap.list_groups().get("brains") or {}).get("members") or []
|
||||
if model_id not in brains:
|
||||
llamaswap.set_group("brains", brains + [model_id], swap=False, persist=True)
|
||||
set_ttl(model_id, 0)
|
||||
except PermissionError as exc:
|
||||
return {"ok": False, "reason": str(exc)}
|
||||
return {"ok": True, "old": old, "new": model_id, "note": "ist bereits das Agent-Hirn"}
|
||||
try:
|
||||
llamaswap.set_role(model_id, "hermes") # 1) Alias
|
||||
brains = (llamaswap.list_groups().get("brains") or {}).get("members") or []
|
||||
new_members = [x for x in brains if x not in (old, model_id)] + [model_id]
|
||||
llamaswap.set_group("brains", new_members, swap=False, persist=True) # 2) warm
|
||||
# 2b) TTL härten: neues Hirn nie auto-entladen; altes Hirn auf Default entspannen.
|
||||
from services.llamaswap import set_ttl, DEFAULT_TTL
|
||||
set_ttl(model_id, 0)
|
||||
if old:
|
||||
set_ttl(old, DEFAULT_TTL)
|
||||
except PermissionError as exc:
|
||||
return {"ok": False, "reason": str(exc)}
|
||||
update_brain_model("hermes") # 3) Config + Restart
|
||||
# Weiche Budget-Warnung (kein Hard-Block): passt Hirn + größtes on-demand zusammen ins GTT?
|
||||
warning = None
|
||||
try:
|
||||
b = hermes_brain_info().get("budget") or {}
|
||||
if b and not b.get("fits", True):
|
||||
warning = (f"Speicher-Warnung: Hirn (~{b.get('brain_gb')} GB) + größtes on-demand-"
|
||||
f"Modell (~{b.get('largest_ondemand_gb')} GB) übersteigen das GTT-Budget "
|
||||
f"(~{b.get('gtt_gb')} GB) — das Hirn ist persistent, heavy/coder laden "
|
||||
f"DANEBEN: Überlauf droht (Lade-Crash/Swapping statt Verdrängung).")
|
||||
except Exception:
|
||||
log.debug("set_agent_brain: Budget-Check fehlgeschlagen", exc_info=True)
|
||||
return {"ok": True, "old": old, "new": model_id, "warning": warning}
|
||||
|
||||
|
||||
def update_brain_model(new_model: str) -> bool:
|
||||
from config import HERMES_HOME
|
||||
home = HERMES_HOME
|
||||
config_path = home / "config.yaml"
|
||||
|
||||
# Ensure home directory exists
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cfg = {}
|
||||
if config_path.exists():
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
except Exception:
|
||||
log.debug("update_brain_model: bestehende config.yaml nicht lesbar", exc_info=True)
|
||||
cfg = {}
|
||||
|
||||
if not isinstance(cfg, dict):
|
||||
cfg = {}
|
||||
|
||||
if "model" not in cfg or not isinstance(cfg["model"], dict):
|
||||
cfg["model"] = {}
|
||||
|
||||
# Hermes liest model.default als aktives Modell; model.model ist der Provider-Param.
|
||||
# Beide setzen, sonst greift die Umschaltung nicht (latenter Bug: nur model.model gesetzt).
|
||||
cfg["model"]["default"] = new_model
|
||||
cfg["model"]["model"] = new_model
|
||||
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("w", encoding="utf-8") as f:
|
||||
r_yaml.dump(cfg, f)
|
||||
|
||||
# Restart the user-space service to apply changes
|
||||
try:
|
||||
import services.maintenance as maintenance
|
||||
maintenance.restart_service("hermes-gateway")
|
||||
except Exception:
|
||||
log.warning("update_brain_model: hermes-gateway-Restart fehlgeschlagen", exc_info=True)
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
log.warning("update_brain_model: Schreiben der config.yaml fehlgeschlagen", exc_info=True)
|
||||
return False
|
||||
"""
|
||||
Hermes-Agent-Status (Control-Plane-Read). MC betreibt Hermes NICHT — es zeigt nur
|
||||
Status + verlinkt das standalone hermes-webui. Voller Zugriff + Tools/MCP werden in
|
||||
Hermes' eigener Config verdrahtet (siehe docs/HERMES_SETUP.md).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
|
||||
from config import (HERMES_TERMINAL_UPSTREAM, HERMES_TERMINAL_PATH, BOX_CONSOLE_UPSTREAM,
|
||||
BOX_CONSOLE_PATH, HERMES_BUILTIN_UI_UPSTREAM, HERMES_BUILTIN_UI_PATH,
|
||||
HERMES_API_URL, HERMES_HOME, PC_EXECUTOR_URL)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _hermes_version(name: str) -> float | None:
|
||||
"""Versionszahl aus 'Hermes-4.3', 'Hermes-4', 'Nous-Hermes-2' → 4.3/4.0/2.0."""
|
||||
low = (name or "").lower()
|
||||
if "hermes" not in low:
|
||||
return None
|
||||
m = re.search(r"hermes[-_ ]?(\d+(?:\.\d+)?)", low)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _active_brain_name() -> str:
|
||||
"""Aktives Agent-Hirn aus Hermes' Config: model.default (sonst model.model)."""
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
p = HERMES_HOME / "config.yaml"
|
||||
if p.exists():
|
||||
with p.open(encoding="utf-8") as f:
|
||||
cfg = YAML().load(f) or {}
|
||||
m = (cfg.get("model") or {}) if isinstance(cfg, dict) else {}
|
||||
return str(m.get("default") or m.get("model") or "auto")
|
||||
except Exception:
|
||||
log.debug("_active_brain_name: Lesefehler", exc_info=True)
|
||||
return "auto"
|
||||
|
||||
|
||||
def hermes_brain_info() -> dict:
|
||||
"""Aktuelles Agent-Hirn = Modell/Alias, das Hermes laut Config nutzt (model.default),
|
||||
plus Budget-Check. Zeigt das REAL genutzte Hirn — unabhängig von einer 'hermes'-Rolle."""
|
||||
from services import llamaswap
|
||||
|
||||
models = llamaswap.list_models()
|
||||
brain = _active_brain_name() # z.B. "fast" (Alias) oder ein Modellname
|
||||
bl = brain.lower()
|
||||
cur = next((m for m in models if (m.get("role") or "").lower() == bl), None) \
|
||||
or next((m for m in models if bl in (m["name"] or "").lower()), None)
|
||||
cur_params = (cur.get("capabilities") or {}).get("params_b") if cur else None
|
||||
current = None
|
||||
if cur:
|
||||
current = {"name": cur["name"], "alias": brain, "filename": cur.get("filename"),
|
||||
"params_b": cur_params, "quant": cur.get("quant"),
|
||||
"size_bytes": cur.get("size_bytes"),
|
||||
"gguf_path": cur.get("gguf_path"), "incomplete": cur.get("incomplete")}
|
||||
|
||||
# Fit-Check: passt das (immer warme) Hirn + das größte on-demand-Modell zusammen ins Budget?
|
||||
budget = None
|
||||
try:
|
||||
from services.budget import footprint_gb, gtt_budget_gb
|
||||
groups = llamaswap.list_groups()
|
||||
persist = set()
|
||||
for g in groups.values():
|
||||
if isinstance(g, dict) and (g.get("persist") or g.get("persistent")):
|
||||
persist.update(g.get("members") or [])
|
||||
|
||||
cur_name = cur["name"] if cur else None
|
||||
brain_gb = footprint_gb(cur) if cur else 0.0
|
||||
# voller Always-Warm-Footprint (alle persist, Brain=Empfehlung) — nur Info
|
||||
warm = brain_gb + sum(footprint_gb(m) for m in models
|
||||
if m["name"] in persist and m["name"] != cur_name)
|
||||
largest_od = max((footprint_gb(m) for m in models if m["name"] not in persist), default=0.0)
|
||||
gtt = gtt_budget_gb()
|
||||
# Seit dem `persistent`-Fix (03.07.) bleibt das GANZE Warmset (Hirn+embed+vision)
|
||||
# resident, wenn ein on-demand-Modell DANEBEN lädt → der reale Peak ist Warmset +
|
||||
# größtes on-demand, nicht nur Hirn + größtes. Genau daran wird `fits` gemessen.
|
||||
budget = {
|
||||
"gtt_gb": gtt,
|
||||
"brain_gb": round(brain_gb, 1),
|
||||
"warm_projected_gb": round(warm, 1),
|
||||
"largest_ondemand_gb": round(largest_od, 1),
|
||||
"fits": (warm + largest_od) <= gtt,
|
||||
"free_after_gb": round(gtt - warm - largest_od, 1),
|
||||
}
|
||||
except Exception:
|
||||
log.debug("hermes_brain_info: Budget-Berechnung fehlgeschlagen", exc_info=True)
|
||||
|
||||
return {"current": current, "recommended": None, "update_available": False, "budget": budget}
|
||||
|
||||
|
||||
def _reach(url: str, path: str = "") -> bool:
|
||||
try:
|
||||
with httpx.Client(timeout=3.0) as c:
|
||||
return c.get(f"{url}{path}").status_code < 500
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
def _count_enabled_mcp_servers() -> int:
|
||||
config_path = HERMES_HOME / "config.yaml"
|
||||
if not config_path.exists():
|
||||
return 0
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
mcp_servers = cfg.get("mcp_servers", {}) if isinstance(cfg, dict) else {}
|
||||
if not isinstance(mcp_servers, dict):
|
||||
return 0
|
||||
return sum(1 for v in mcp_servers.values() if isinstance(v, dict) and v.get("enabled", True))
|
||||
except Exception:
|
||||
log.debug("_count_enabled_mcp_servers: Fehler", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def agent_status() -> dict:
|
||||
"""Erreichbarkeit von Gateway (:8642) + WebUI (:8787) + lokale Hinweise."""
|
||||
home = HERMES_HOME
|
||||
brain_model = "auto"
|
||||
config_path = home / "config.yaml"
|
||||
if config_path.exists():
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
if isinstance(cfg, dict):
|
||||
# Hermes nutzt model.default als aktives Modell (model.model = Provider-Param).
|
||||
m = cfg.get("model", {}) or {}
|
||||
brain_model = m.get("default") or m.get("model") or "auto"
|
||||
except Exception:
|
||||
log.debug("agent_status: Hermes-config.yaml nicht lesbar", exc_info=True)
|
||||
|
||||
|
||||
return {
|
||||
"gateway_url": HERMES_API_URL,
|
||||
# Interaktives Web-Terminal (ttyd → `hermes chat`): same-origin über MC2 geproxyt.
|
||||
"terminal_url": HERMES_TERMINAL_PATH,
|
||||
# Box-Konsole (ttyd → Login-Shell): same-origin über MC2 geproxyt (/console/).
|
||||
"box_console_url": BOX_CONSOLE_PATH,
|
||||
# Eingebaute Hermes-Web-GUI (hermes serve): same-origin über MC2 geproxyt (/hermes-ui/).
|
||||
"hermes_ui_url": HERMES_BUILTIN_UI_PATH,
|
||||
"gateway_reachable": _reach(HERMES_API_URL, "/health"),
|
||||
# Erreichbarkeit der lokalen ttyd-Upstreams (Loopback, je base-path).
|
||||
"terminal_reachable": _reach(HERMES_TERMINAL_UPSTREAM, "/hermes-terminal/"),
|
||||
"box_console_reachable": _reach(BOX_CONSOLE_UPSTREAM, "/console/"),
|
||||
# Erreichbarkeit der eingebauten Hermes-GUI (Loopback :9119).
|
||||
"hermes_ui_reachable": _reach(HERMES_BUILTIN_UI_UPSTREAM, "/"),
|
||||
"home_exists": home.exists(),
|
||||
"brain_model": brain_model,
|
||||
# Best-effort: welche Verdrahtung lokal sichtbar ist (auf der Box aussagekräftig).
|
||||
"has_config": (home / "config.yaml").exists() or (home / "config.json").exists(),
|
||||
"has_skills": (home / "skills").exists(),
|
||||
"has_memories": (home / "memories").exists(),
|
||||
# Neue Felder: Telegram, MCP-Server-Anzahl, PC-Executor-Erreichbarkeit.
|
||||
"telegram_enabled": bool(os.environ.get("TELEGRAM_BOT_TOKEN", "")),
|
||||
"mcp_server_count": _count_enabled_mcp_servers(),
|
||||
"pc_executor_reachable": _reach(PC_EXECUTOR_URL, "/health"),
|
||||
}
|
||||
|
||||
|
||||
def set_agent_brain(model_id: str) -> dict:
|
||||
"""Setzt ein (bereits installiertes) Modell als Agent-Hirn — WARM-bewusst:
|
||||
1) vergibt den 'hermes'-Alias (das Agent-Hirn-Slot),
|
||||
2) tauscht es in die residente brains-Gruppe (altes Hirn raus, fast/vision bleiben),
|
||||
3) zeigt die Hermes-Config auf den 'hermes'-Alias + Gateway-Restart.
|
||||
So bleibt das neue Hirn warm und der Agent nutzt es sofort."""
|
||||
from services import llamaswap
|
||||
models = {m["name"]: m for m in llamaswap.list_models()}
|
||||
if model_id not in models:
|
||||
return {"ok": False, "reason": "Modell nicht installiert — erst über Modelle-finden laden."}
|
||||
old = next((m["name"] for m in models.values() if m.get("role") == "hermes"), None)
|
||||
if model_id == old:
|
||||
# Idempotent härten: auch wenn schon Hirn, warm (brains) + ttl 0 sicherstellen.
|
||||
try:
|
||||
from services.llamaswap import set_ttl
|
||||
brains = (llamaswap.list_groups().get("brains") or {}).get("members") or []
|
||||
if model_id not in brains:
|
||||
llamaswap.set_group("brains", brains + [model_id], swap=False, persist=True)
|
||||
set_ttl(model_id, 0)
|
||||
except PermissionError as exc:
|
||||
return {"ok": False, "reason": str(exc)}
|
||||
return {"ok": True, "old": old, "new": model_id, "note": "ist bereits das Agent-Hirn"}
|
||||
try:
|
||||
llamaswap.set_role(model_id, "hermes") # 1) Alias
|
||||
brains = (llamaswap.list_groups().get("brains") or {}).get("members") or []
|
||||
new_members = [x for x in brains if x not in (old, model_id)] + [model_id]
|
||||
llamaswap.set_group("brains", new_members, swap=False, persist=True) # 2) warm
|
||||
# 2b) TTL härten: neues Hirn nie auto-entladen; altes Hirn auf Default entspannen.
|
||||
from services.llamaswap import set_ttl, DEFAULT_TTL
|
||||
set_ttl(model_id, 0)
|
||||
if old:
|
||||
set_ttl(old, DEFAULT_TTL)
|
||||
except PermissionError as exc:
|
||||
return {"ok": False, "reason": str(exc)}
|
||||
update_brain_model("hermes") # 3) Config + Restart
|
||||
# Weiche Budget-Warnung (kein Hard-Block): passt Hirn + größtes on-demand zusammen ins GTT?
|
||||
warning = None
|
||||
try:
|
||||
b = hermes_brain_info().get("budget") or {}
|
||||
if b and not b.get("fits", True):
|
||||
warning = (f"Speicher-Warnung: Hirn (~{b.get('brain_gb')} GB) + größtes on-demand-"
|
||||
f"Modell (~{b.get('largest_ondemand_gb')} GB) übersteigen das GTT-Budget "
|
||||
f"(~{b.get('gtt_gb')} GB) — das Hirn ist persistent, heavy/coder laden "
|
||||
f"DANEBEN: Überlauf droht (Lade-Crash/Swapping statt Verdrängung).")
|
||||
except Exception:
|
||||
log.debug("set_agent_brain: Budget-Check fehlgeschlagen", exc_info=True)
|
||||
return {"ok": True, "old": old, "new": model_id, "warning": warning}
|
||||
|
||||
|
||||
def update_brain_model(new_model: str) -> bool:
|
||||
from config import HERMES_HOME
|
||||
home = HERMES_HOME
|
||||
config_path = home / "config.yaml"
|
||||
|
||||
# Ensure home directory exists
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cfg = {}
|
||||
if config_path.exists():
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
cfg = r_yaml.load(f) or {}
|
||||
except Exception:
|
||||
log.debug("update_brain_model: bestehende config.yaml nicht lesbar", exc_info=True)
|
||||
cfg = {}
|
||||
|
||||
if not isinstance(cfg, dict):
|
||||
cfg = {}
|
||||
|
||||
if "model" not in cfg or not isinstance(cfg["model"], dict):
|
||||
cfg["model"] = {}
|
||||
|
||||
# Hermes liest model.default als aktives Modell; model.model ist der Provider-Param.
|
||||
# Beide setzen, sonst greift die Umschaltung nicht (latenter Bug: nur model.model gesetzt).
|
||||
cfg["model"]["default"] = new_model
|
||||
cfg["model"]["model"] = new_model
|
||||
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
r_yaml = YAML()
|
||||
with config_path.open("w", encoding="utf-8") as f:
|
||||
r_yaml.dump(cfg, f)
|
||||
|
||||
# Restart the user-space service to apply changes
|
||||
try:
|
||||
import services.maintenance as maintenance
|
||||
maintenance.restart_service("hermes-gateway")
|
||||
except Exception:
|
||||
log.warning("update_brain_model: hermes-gateway-Restart fehlgeschlagen", exc_info=True)
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
log.warning("update_brain_model: Schreiben der config.yaml fehlgeschlagen", exc_info=True)
|
||||
return False
|
||||
|
||||
+205
-205
@@ -1,205 +1,205 @@
|
||||
"""
|
||||
Speicher-Budget & SETUP-BEWUSSTE ctx-Vergabe — EINE Quelle der Wahrheit.
|
||||
|
||||
Modelliert die auf der Box VERIFIZIERTE Residenz-Realität (llama-swap, GTT ~124 GB):
|
||||
• Die `persistent`-Gruppe (brains = Hirn+embed+vision) bleibt IMMER resident —
|
||||
seit dem Key-Fix 03.07.2026 greift der Schutz wirklich (vorher stand `persist`
|
||||
in der Config, das llama-swap stillschweigend ignorierte; on-demand-Last
|
||||
verdrängte damals die ganze Gruppe).
|
||||
• Ein on-demand-Modell (heavy/coder/…) lädt NEBEN die brains-Gruppe und muss
|
||||
deren Footprint mit einplanen.
|
||||
Daraus folgt, wie viel Speicher NEBEN einem Zielmodell reserviert bleiben muss —
|
||||
und damit der größte Kontext, der wirklich passt (nicht nur für das Modell allein).
|
||||
|
||||
Vorher rechnete nur der Hirn-Wechsel (agent.py) setup-bewusst; die allgemeine
|
||||
ctx-Vergabe nahm den Gesamt-RAM in Isolation. Dieses Modul vereint beides.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import psutil
|
||||
|
||||
from services.fit import (
|
||||
QUANT_BYTES_PER_PARAM,
|
||||
estimate_memory_gb,
|
||||
extract_params_b,
|
||||
max_ctx_in_budget,
|
||||
)
|
||||
|
||||
HEADROOM_GB = 4.0 # OS/Treiber/Fragmentierung
|
||||
|
||||
|
||||
def gtt_budget_gb() -> float:
|
||||
"""GPU-adressierbarer Speicher (GTT) in GB — die harte Obergrenze. Liest
|
||||
amdgpu.gttsize aus /proc/cmdline, sonst RAM minus OS-Reserve."""
|
||||
try:
|
||||
with open("/proc/cmdline") as f:
|
||||
m = re.search(r"amdgpu\.gttsize=(\d+)", f.read())
|
||||
if m:
|
||||
return round(int(m.group(1)) / 1024.0, 1)
|
||||
except Exception:
|
||||
pass
|
||||
return round(psutil.virtual_memory().total / (1024 ** 3) - 6.0, 1)
|
||||
|
||||
|
||||
def params_of_model(model: dict) -> float:
|
||||
"""Robuste Params (Mrd.) eines INSTALLIERTEN Modells: MAXIMUM aus Caps-Schätzung und
|
||||
Dateigröße. Deckt 'Coder-Next' ohne Größe im Namen (→ aus Datei) und Split-GGUFs
|
||||
(size_bytes = nur erster Teil → ignoriert) ab."""
|
||||
caps = model.get("capabilities") or {}
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
pb_size = (size_gb / bpp) if size_gb > 1.0 else 0.0
|
||||
return max(float(caps.get("params_b") or 0), pb_size, 7.0)
|
||||
|
||||
|
||||
_CTK_RE = re.compile(r"(?:--cache-type-k|(?<![\w-])-ctk)\s+(\S+)")
|
||||
_CTV_RE = re.compile(r"(?:--cache-type-v|(?<![\w-])-ctv)\s+(\S+)")
|
||||
|
||||
|
||||
def _cache_types(cmd: str) -> tuple[str | None, str | None]:
|
||||
"""K/V-Cache-Quantisierung aus dem llama-server-Cmd (Default f16 → None)."""
|
||||
ck = m.group(1) if (m := _CTK_RE.search(cmd or "")) else None
|
||||
cv = m.group(1) if (m := _CTV_RE.search(cmd or "")) else None
|
||||
return ck, cv
|
||||
|
||||
|
||||
def _real_kv_gb(model: dict, ctx: int) -> float | None:
|
||||
"""ECHTE KV-Cache-Größe (GiB) aus den GGUF-Architektur-Metadaten (Layer × KV-Heads ×
|
||||
Head-Dim) + der cache-type-Quantisierung des Cmds. None, wenn das GGUF nicht lesbar ist
|
||||
→ Aufrufer fällt auf die params-basierte Heuristik zurück."""
|
||||
from services import gguf_meta
|
||||
path = model.get("gguf_path")
|
||||
if not path:
|
||||
return None
|
||||
meta = gguf_meta.arch_meta(path)
|
||||
if not meta:
|
||||
return None
|
||||
ck, cv = _cache_types(model.get("cmd") or "")
|
||||
return gguf_meta.kv_cache_gb(meta, ctx, ck, cv)
|
||||
|
||||
|
||||
def footprint_gb(model: dict) -> float:
|
||||
"""Loaded-Footprint eines Modells = Gewichte + KV-Cache (bei seinem aktuellen ctx).
|
||||
KV kommt aus den ECHTEN Architektur-Metadaten des GGUF (nicht mehr params-geschätzt) —
|
||||
entscheidend bei MoE (A3B): die alte Schätzung hing an den Gesamt-Params und überschätzte
|
||||
grob (z.B. „68 GB reserviert" statt real ~25 GB). Heuristik bleibt Fallback."""
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
ctx = int(model.get("ctx") or 32768)
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
pb = params_of_model(model)
|
||||
weights = max(pb * bpp, size_gb)
|
||||
kv = _real_kv_gb(model, ctx)
|
||||
if kv is None:
|
||||
kv = estimate_memory_gb(pb, quant, ctx) - pb * bpp
|
||||
return weights + max(kv, 0.0)
|
||||
|
||||
|
||||
def params_b_for(name: str) -> float:
|
||||
"""Parameter (Mrd.) für einen Modell-/Repo-Namen: KATALOG (echte Metadaten) zuerst,
|
||||
sonst Namens-Schätzung. Gemeinsam für Fit-Vorschau und ctx-Vergabe."""
|
||||
from services import catalog
|
||||
meta = catalog.meta_for_name(name) if name else None
|
||||
if meta and meta.get("total_params_b"):
|
||||
return float(meta["total_params_b"])
|
||||
return extract_params_b(name)
|
||||
|
||||
|
||||
def _coresident_members(groups: dict) -> set:
|
||||
"""Modelle, die GLEICHZEITIG warm sind: Mitglieder aller `swap:false`-Gruppen
|
||||
(Ko-Residenz, z.B. brains = Hirn+embed+vision). Seit dem `persistent`-Fix (03.07.2026)
|
||||
überlebt die Gruppe auch on-demand-Last: Coder/heavy laden DANEBEN, nicht an ihre
|
||||
Stelle (live verifiziert: Coder + Qwen3.6 gleichzeitig `ready`). Die frühere
|
||||
Beobachtung „heavy verdrängt die Gruppe" war der ignorierte `persist`-Key."""
|
||||
out: set = set()
|
||||
for g in (groups or {}).values():
|
||||
if isinstance(g, dict) and g.get("swap") is False:
|
||||
out.update(g.get("members") or [])
|
||||
return out
|
||||
|
||||
|
||||
def reserved_gb(role: str | None) -> dict:
|
||||
"""Speicher, der NEBEN einem Zielmodell der gegebenen Rolle resident bleibt — gemäß der
|
||||
seit dem `persistent`-Fix (03.07.2026) geltenden Semantik: die ko-residente
|
||||
`swap:false`-Gruppe (brains) bleibt IMMER geladen, on-demand-Modelle laden daneben.
|
||||
|
||||
- Modell IN der Ko-Residenz-Gruppe (Hirn/embed/vision): koexistiert mit den ÜBRIGEN
|
||||
Gruppen-Mitgliedern → reserviert deren Summe.
|
||||
- Modell AUSSERHALB (heavy/coder/coder-lite/scout): lädt NEBEN die Gruppe →
|
||||
reserviert deren GESAMTE Summe (früher 0.0, weil der kaputte `persist`-Key die
|
||||
Gruppe verdrängen ließ — diese Rechnung erlaubte zu große Kontexte).
|
||||
"""
|
||||
from services import llamaswap
|
||||
models = llamaswap.list_models()
|
||||
groups = llamaswap.list_groups()
|
||||
cores = _coresident_members(groups)
|
||||
brain = next((m for m in models if (m.get("role") == "hermes")), None)
|
||||
brain_gb = footprint_gb(brain) if brain else 0.0
|
||||
role = (role or "").strip().lower()
|
||||
|
||||
holder = next((m for m in models if (m.get("role") == role)), None) if role else None
|
||||
holder_name = holder["name"] if holder else None
|
||||
# Hirn (hermes) ist per Definition Teil der Ko-Residenz-Gruppe; sonst Gruppen-Mitgliedschaft prüfen.
|
||||
in_group = role == "hermes" or bool(holder_name and holder_name in cores)
|
||||
|
||||
if in_group:
|
||||
others = sum(footprint_gb(m) for m in models
|
||||
if m["name"] in cores and m["name"] != holder_name)
|
||||
return {"reserved_gb": others, "mode": "co-resident", "brain_gb": brain_gb}
|
||||
# on-demand: lädt neben die (persistente) Ko-Residenz-Gruppe → deren Summe reservieren.
|
||||
warm = sum(footprint_gb(m) for m in models if m["name"] in cores)
|
||||
return {"reserved_gb": warm, "mode": "ondemand-beside-warmset", "brain_gb": brain_gb}
|
||||
|
||||
|
||||
def setup_aware_ctx(params_b: float, quant: str, role: str | None = None) -> dict:
|
||||
"""Größter Kontext, der für ein Modell (params_b/quant) der gegebenen Rolle NEBEN dem
|
||||
bestehenden Setup passt. Gibt ctx + die Budget-Herleitung zurück (für UI/Transparenz)."""
|
||||
gtt = gtt_budget_gb()
|
||||
r = reserved_gb(role)
|
||||
budget = max(gtt - r["reserved_gb"] - HEADROOM_GB, 0.0)
|
||||
ctx = max_ctx_in_budget(params_b, quant, budget)
|
||||
return {
|
||||
"ctx": ctx,
|
||||
"gtt_gb": gtt,
|
||||
"reserved_gb": round(r["reserved_gb"], 1),
|
||||
"budget_gb": round(budget, 1),
|
||||
"mode": r["mode"],
|
||||
}
|
||||
|
||||
|
||||
def _snap_ctx(raw_ctx: float, cap: int | None = None) -> int:
|
||||
"""Größter 'schöner' Kontext ≤ raw_ctx (und ≤ Trainings-Kontext des Modells, falls bekannt)."""
|
||||
from services.fit import _NICE_CTX
|
||||
if cap:
|
||||
raw_ctx = min(raw_ctx, cap)
|
||||
best = _NICE_CTX[0]
|
||||
for c in _NICE_CTX:
|
||||
if c <= raw_ctx:
|
||||
best = c
|
||||
return best
|
||||
|
||||
|
||||
def setup_aware_ctx_for_model(model: dict) -> dict:
|
||||
"""Setup-bewusster Optimal-ctx für ein INSTALLIERTES Modell. Für den 'Auto'-Button an der
|
||||
Modellkarte. Nutzt die ECHTE KV-Größe des GGUF (gleiche Zahlensprache wie footprint_gb) —
|
||||
Fallback auf die params-Heuristik nur, wenn das GGUF nicht lesbar ist."""
|
||||
from services import gguf_meta
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
path = model.get("gguf_path")
|
||||
meta = gguf_meta.arch_meta(path) if path else None
|
||||
if not meta:
|
||||
return setup_aware_ctx(params_of_model(model), quant, role=model.get("role"))
|
||||
|
||||
gtt = gtt_budget_gb()
|
||||
r = reserved_gb(model.get("role"))
|
||||
budget = max(gtt - r["reserved_gb"] - HEADROOM_GB, 0.0)
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
weights = max(params_of_model(model) * bpp, size_gb)
|
||||
ck, cv = _cache_types(model.get("cmd") or "")
|
||||
per_tok = gguf_meta.kv_gb_per_token(meta, ck, cv)
|
||||
ctx = _snap_ctx((budget - weights) / per_tok, cap=meta.get("n_ctx_train")) if per_tok > 0 else 2048
|
||||
return {"ctx": ctx, "gtt_gb": gtt, "reserved_gb": round(r["reserved_gb"], 1),
|
||||
"budget_gb": round(budget, 1), "mode": r["mode"]}
|
||||
"""
|
||||
Speicher-Budget & SETUP-BEWUSSTE ctx-Vergabe — EINE Quelle der Wahrheit.
|
||||
|
||||
Modelliert die auf der Box VERIFIZIERTE Residenz-Realität (llama-swap, GTT ~124 GB):
|
||||
• Die `persistent`-Gruppe (brains = Hirn+embed+vision) bleibt IMMER resident —
|
||||
seit dem Key-Fix 03.07.2026 greift der Schutz wirklich (vorher stand `persist`
|
||||
in der Config, das llama-swap stillschweigend ignorierte; on-demand-Last
|
||||
verdrängte damals die ganze Gruppe).
|
||||
• Ein on-demand-Modell (heavy/coder/…) lädt NEBEN die brains-Gruppe und muss
|
||||
deren Footprint mit einplanen.
|
||||
Daraus folgt, wie viel Speicher NEBEN einem Zielmodell reserviert bleiben muss —
|
||||
und damit der größte Kontext, der wirklich passt (nicht nur für das Modell allein).
|
||||
|
||||
Vorher rechnete nur der Hirn-Wechsel (agent.py) setup-bewusst; die allgemeine
|
||||
ctx-Vergabe nahm den Gesamt-RAM in Isolation. Dieses Modul vereint beides.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import psutil
|
||||
|
||||
from services.fit import (
|
||||
QUANT_BYTES_PER_PARAM,
|
||||
estimate_memory_gb,
|
||||
extract_params_b,
|
||||
max_ctx_in_budget,
|
||||
)
|
||||
|
||||
HEADROOM_GB = 4.0 # OS/Treiber/Fragmentierung
|
||||
|
||||
|
||||
def gtt_budget_gb() -> float:
|
||||
"""GPU-adressierbarer Speicher (GTT) in GB — die harte Obergrenze. Liest
|
||||
amdgpu.gttsize aus /proc/cmdline, sonst RAM minus OS-Reserve."""
|
||||
try:
|
||||
with open("/proc/cmdline") as f:
|
||||
m = re.search(r"amdgpu\.gttsize=(\d+)", f.read())
|
||||
if m:
|
||||
return round(int(m.group(1)) / 1024.0, 1)
|
||||
except Exception:
|
||||
pass
|
||||
return round(psutil.virtual_memory().total / (1024 ** 3) - 6.0, 1)
|
||||
|
||||
|
||||
def params_of_model(model: dict) -> float:
|
||||
"""Robuste Params (Mrd.) eines INSTALLIERTEN Modells: MAXIMUM aus Caps-Schätzung und
|
||||
Dateigröße. Deckt 'Coder-Next' ohne Größe im Namen (→ aus Datei) und Split-GGUFs
|
||||
(size_bytes = nur erster Teil → ignoriert) ab."""
|
||||
caps = model.get("capabilities") or {}
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
pb_size = (size_gb / bpp) if size_gb > 1.0 else 0.0
|
||||
return max(float(caps.get("params_b") or 0), pb_size, 7.0)
|
||||
|
||||
|
||||
_CTK_RE = re.compile(r"(?:--cache-type-k|(?<![\w-])-ctk)\s+(\S+)")
|
||||
_CTV_RE = re.compile(r"(?:--cache-type-v|(?<![\w-])-ctv)\s+(\S+)")
|
||||
|
||||
|
||||
def _cache_types(cmd: str) -> tuple[str | None, str | None]:
|
||||
"""K/V-Cache-Quantisierung aus dem llama-server-Cmd (Default f16 → None)."""
|
||||
ck = m.group(1) if (m := _CTK_RE.search(cmd or "")) else None
|
||||
cv = m.group(1) if (m := _CTV_RE.search(cmd or "")) else None
|
||||
return ck, cv
|
||||
|
||||
|
||||
def _real_kv_gb(model: dict, ctx: int) -> float | None:
|
||||
"""ECHTE KV-Cache-Größe (GiB) aus den GGUF-Architektur-Metadaten (Layer × KV-Heads ×
|
||||
Head-Dim) + der cache-type-Quantisierung des Cmds. None, wenn das GGUF nicht lesbar ist
|
||||
→ Aufrufer fällt auf die params-basierte Heuristik zurück."""
|
||||
from services import gguf_meta
|
||||
path = model.get("gguf_path")
|
||||
if not path:
|
||||
return None
|
||||
meta = gguf_meta.arch_meta(path)
|
||||
if not meta:
|
||||
return None
|
||||
ck, cv = _cache_types(model.get("cmd") or "")
|
||||
return gguf_meta.kv_cache_gb(meta, ctx, ck, cv)
|
||||
|
||||
|
||||
def footprint_gb(model: dict) -> float:
|
||||
"""Loaded-Footprint eines Modells = Gewichte + KV-Cache (bei seinem aktuellen ctx).
|
||||
KV kommt aus den ECHTEN Architektur-Metadaten des GGUF (nicht mehr params-geschätzt) —
|
||||
entscheidend bei MoE (A3B): die alte Schätzung hing an den Gesamt-Params und überschätzte
|
||||
grob (z.B. „68 GB reserviert" statt real ~25 GB). Heuristik bleibt Fallback."""
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
ctx = int(model.get("ctx") or 32768)
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
pb = params_of_model(model)
|
||||
weights = max(pb * bpp, size_gb)
|
||||
kv = _real_kv_gb(model, ctx)
|
||||
if kv is None:
|
||||
kv = estimate_memory_gb(pb, quant, ctx) - pb * bpp
|
||||
return weights + max(kv, 0.0)
|
||||
|
||||
|
||||
def params_b_for(name: str) -> float:
|
||||
"""Parameter (Mrd.) für einen Modell-/Repo-Namen: KATALOG (echte Metadaten) zuerst,
|
||||
sonst Namens-Schätzung. Gemeinsam für Fit-Vorschau und ctx-Vergabe."""
|
||||
from services import catalog
|
||||
meta = catalog.meta_for_name(name) if name else None
|
||||
if meta and meta.get("total_params_b"):
|
||||
return float(meta["total_params_b"])
|
||||
return extract_params_b(name)
|
||||
|
||||
|
||||
def _coresident_members(groups: dict) -> set:
|
||||
"""Modelle, die GLEICHZEITIG warm sind: Mitglieder aller `swap:false`-Gruppen
|
||||
(Ko-Residenz, z.B. brains = Hirn+embed+vision). Seit dem `persistent`-Fix (03.07.2026)
|
||||
überlebt die Gruppe auch on-demand-Last: Coder/heavy laden DANEBEN, nicht an ihre
|
||||
Stelle (live verifiziert: Coder + Qwen3.6 gleichzeitig `ready`). Die frühere
|
||||
Beobachtung „heavy verdrängt die Gruppe" war der ignorierte `persist`-Key."""
|
||||
out: set = set()
|
||||
for g in (groups or {}).values():
|
||||
if isinstance(g, dict) and g.get("swap") is False:
|
||||
out.update(g.get("members") or [])
|
||||
return out
|
||||
|
||||
|
||||
def reserved_gb(role: str | None) -> dict:
|
||||
"""Speicher, der NEBEN einem Zielmodell der gegebenen Rolle resident bleibt — gemäß der
|
||||
seit dem `persistent`-Fix (03.07.2026) geltenden Semantik: die ko-residente
|
||||
`swap:false`-Gruppe (brains) bleibt IMMER geladen, on-demand-Modelle laden daneben.
|
||||
|
||||
- Modell IN der Ko-Residenz-Gruppe (Hirn/embed/vision): koexistiert mit den ÜBRIGEN
|
||||
Gruppen-Mitgliedern → reserviert deren Summe.
|
||||
- Modell AUSSERHALB (heavy/coder/coder-lite/scout): lädt NEBEN die Gruppe →
|
||||
reserviert deren GESAMTE Summe (früher 0.0, weil der kaputte `persist`-Key die
|
||||
Gruppe verdrängen ließ — diese Rechnung erlaubte zu große Kontexte).
|
||||
"""
|
||||
from services import llamaswap
|
||||
models = llamaswap.list_models()
|
||||
groups = llamaswap.list_groups()
|
||||
cores = _coresident_members(groups)
|
||||
brain = next((m for m in models if (m.get("role") == "hermes")), None)
|
||||
brain_gb = footprint_gb(brain) if brain else 0.0
|
||||
role = (role or "").strip().lower()
|
||||
|
||||
holder = next((m for m in models if (m.get("role") == role)), None) if role else None
|
||||
holder_name = holder["name"] if holder else None
|
||||
# Hirn (hermes) ist per Definition Teil der Ko-Residenz-Gruppe; sonst Gruppen-Mitgliedschaft prüfen.
|
||||
in_group = role == "hermes" or bool(holder_name and holder_name in cores)
|
||||
|
||||
if in_group:
|
||||
others = sum(footprint_gb(m) for m in models
|
||||
if m["name"] in cores and m["name"] != holder_name)
|
||||
return {"reserved_gb": others, "mode": "co-resident", "brain_gb": brain_gb}
|
||||
# on-demand: lädt neben die (persistente) Ko-Residenz-Gruppe → deren Summe reservieren.
|
||||
warm = sum(footprint_gb(m) for m in models if m["name"] in cores)
|
||||
return {"reserved_gb": warm, "mode": "ondemand-beside-warmset", "brain_gb": brain_gb}
|
||||
|
||||
|
||||
def setup_aware_ctx(params_b: float, quant: str, role: str | None = None) -> dict:
|
||||
"""Größter Kontext, der für ein Modell (params_b/quant) der gegebenen Rolle NEBEN dem
|
||||
bestehenden Setup passt. Gibt ctx + die Budget-Herleitung zurück (für UI/Transparenz)."""
|
||||
gtt = gtt_budget_gb()
|
||||
r = reserved_gb(role)
|
||||
budget = max(gtt - r["reserved_gb"] - HEADROOM_GB, 0.0)
|
||||
ctx = max_ctx_in_budget(params_b, quant, budget)
|
||||
return {
|
||||
"ctx": ctx,
|
||||
"gtt_gb": gtt,
|
||||
"reserved_gb": round(r["reserved_gb"], 1),
|
||||
"budget_gb": round(budget, 1),
|
||||
"mode": r["mode"],
|
||||
}
|
||||
|
||||
|
||||
def _snap_ctx(raw_ctx: float, cap: int | None = None) -> int:
|
||||
"""Größter 'schöner' Kontext ≤ raw_ctx (und ≤ Trainings-Kontext des Modells, falls bekannt)."""
|
||||
from services.fit import _NICE_CTX
|
||||
if cap:
|
||||
raw_ctx = min(raw_ctx, cap)
|
||||
best = _NICE_CTX[0]
|
||||
for c in _NICE_CTX:
|
||||
if c <= raw_ctx:
|
||||
best = c
|
||||
return best
|
||||
|
||||
|
||||
def setup_aware_ctx_for_model(model: dict) -> dict:
|
||||
"""Setup-bewusster Optimal-ctx für ein INSTALLIERTES Modell. Für den 'Auto'-Button an der
|
||||
Modellkarte. Nutzt die ECHTE KV-Größe des GGUF (gleiche Zahlensprache wie footprint_gb) —
|
||||
Fallback auf die params-Heuristik nur, wenn das GGUF nicht lesbar ist."""
|
||||
from services import gguf_meta
|
||||
quant = model.get("quant") or "Q4_K_M"
|
||||
path = model.get("gguf_path")
|
||||
meta = gguf_meta.arch_meta(path) if path else None
|
||||
if not meta:
|
||||
return setup_aware_ctx(params_of_model(model), quant, role=model.get("role"))
|
||||
|
||||
gtt = gtt_budget_gb()
|
||||
r = reserved_gb(model.get("role"))
|
||||
budget = max(gtt - r["reserved_gb"] - HEADROOM_GB, 0.0)
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant.upper(), 0.55)
|
||||
size_gb = (model.get("size_bytes") or 0) / (1024 ** 3)
|
||||
weights = max(params_of_model(model) * bpp, size_gb)
|
||||
ck, cv = _cache_types(model.get("cmd") or "")
|
||||
per_tok = gguf_meta.kv_gb_per_token(meta, ck, cv)
|
||||
ctx = _snap_ctx((budget - weights) / per_tok, cap=meta.get("n_ctx_train")) if per_tok > 0 else 2048
|
||||
return {"ctx": ctx, "gtt_gb": gtt, "reserved_gb": round(r["reserved_gb"], 1),
|
||||
"budget_gb": round(budget, 1), "mode": r["mode"]}
|
||||
|
||||
+179
-179
@@ -1,179 +1,179 @@
|
||||
"""
|
||||
Automatische Modell-Entdeckung ("aktuell beste Modelle"): fragt vertrauenswürdige
|
||||
HF-Orgs live ab, kategorisiert per Stichwort, rankt nach Hardware-Fit + Beliebtheit
|
||||
und cached. Portiert aus Mission Control v1 (cookbook.py-Discover).
|
||||
|
||||
Wichtig (Greenfield-Fix gegen v1): EIN gemeinsamer Ranking-Helfer `rank_runnable`
|
||||
ist die Quelle der Wahrheit — sowohl die „beste Empfehlung" je Kategorie als auch
|
||||
spätere Auto-Setups nutzen ihn, damit sie nie auseinanderlaufen.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
|
||||
import logging
|
||||
|
||||
from config import DISCOVER_CACHE_PATH, DISCOVER_TTL
|
||||
from services import catalog
|
||||
from services.caps import capabilities
|
||||
from services.fit import evaluate_fit, extract_params_b, max_ctx_for
|
||||
from services.sources import CATEGORIES, SKIP_TOKENS, TRUSTED_AUTHORS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_FIT_ORDER = {"perfect": 0, "marginal": 1, "too_tight": 2}
|
||||
|
||||
|
||||
def _categorize(repo_id: str) -> str:
|
||||
low = repo_id.lower()
|
||||
for cat in CATEGORIES:
|
||||
if any(k in low for k in cat["kw"]):
|
||||
return cat["role"]
|
||||
return "scout"
|
||||
|
||||
|
||||
def _fetch_author_models(author: str) -> list:
|
||||
url = (f"https://huggingface.co/api/models?author={author}"
|
||||
f"&filter=gguf&sort=downloads&direction=-1&limit=40")
|
||||
try:
|
||||
with httpx.Client(timeout=12.0) as c:
|
||||
data = c.get(url).json()
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
log.debug("discover: Abfrage für Autor %s fehlgeschlagen", author, exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
def _age_days(last_modified, now_ts: float) -> float:
|
||||
"""Alter eines HF-Modells in Tagen (lastModified ISO). Unbekannt → ~1.5 Jahre."""
|
||||
if not last_modified:
|
||||
return 540.0
|
||||
try:
|
||||
dt = datetime.fromisoformat(str(last_modified).replace("Z", "+00:00"))
|
||||
return max((now_ts - dt.timestamp()) / 86400.0, 0.0)
|
||||
except Exception:
|
||||
return 540.0
|
||||
|
||||
|
||||
def _score(m: dict, now_ts: float) -> float:
|
||||
"""Zukunftssicherer Rang-Score für DIESE Hardware. Kombiniert:
|
||||
- Fit: perfect dominiert (Bonus 3.0 > Summe der übrigen Terme → passt-komfortabel zuerst),
|
||||
- Recency: neuere Generationen bevorzugt (Halbwertszeit ~9 Monate über lastModified),
|
||||
- Capability: mehr Parameter (log-skaliert),
|
||||
- Popularity: Downloads (log-skaliert).
|
||||
So gewinnt bei vergleichbarer Größe die NEUERE Generation (z.B. Qwen3-Coder vor
|
||||
Qwen2.5-Coder), ohne dass kleine Populär-Modelle große verdrängen."""
|
||||
fit_bonus = 3.0 if m["fit"]["level"] == "perfect" else 0.0
|
||||
recency = 0.5 ** (_age_days(m.get("lastModified"), now_ts) / 270.0)
|
||||
cap = math.log2(max(float(m.get("params_b") or 1.0), 1.0) + 1.0) / 8.0
|
||||
pop = math.log10(float(m.get("downloads") or 0) + 1.0) / 7.0
|
||||
return fit_bonus + 1.2 * recency + 1.2 * cap + 0.5 * pop
|
||||
|
||||
|
||||
def rank_runnable(models: list[dict]) -> list[dict]:
|
||||
"""EINE Quelle der Wahrheit fürs Ranking lauffähiger Modelle für DIESE Hardware.
|
||||
Nur was passt (too_tight fliegt raus), dann nach `_score` (Fit + Recency + Capability
|
||||
+ Popularity). Bevorzugt neuere, fähige Modelle → zukunftssicher; „Modelle finden"
|
||||
schlägt nie ein Downgrade vor (Downgrade-Sperre zusätzlich in maintenance)."""
|
||||
now_ts = time.time()
|
||||
return sorted(
|
||||
[m for m in models if m["fit"]["level"] != "too_tight"],
|
||||
key=lambda m: -_score(m, now_ts),
|
||||
)
|
||||
|
||||
|
||||
def refresh_discover(ram_gb: float) -> dict:
|
||||
"""Quellen live abfragen, kategorisieren, ranken, cachen. Wirft nur, wenn KEINE
|
||||
Quelle erreichbar war."""
|
||||
raw, seen, ok = [], set(), 0
|
||||
for author in TRUSTED_AUTHORS:
|
||||
models = _fetch_author_models(author)
|
||||
if models:
|
||||
ok += 1
|
||||
for m in models:
|
||||
rid = m.get("id")
|
||||
if not rid or rid in seen:
|
||||
continue
|
||||
seen.add(rid)
|
||||
raw.append(m)
|
||||
if ok == 0 and not catalog.entries():
|
||||
raise RuntimeError("Keine Quelle erreichbar.")
|
||||
|
||||
by_cat: dict[str, list] = {c["role"]: [] for c in CATEGORIES}
|
||||
for m in raw:
|
||||
rid = m["id"]
|
||||
low = rid.lower()
|
||||
if any(tok in low for tok in SKIP_TOKENS):
|
||||
continue
|
||||
role = _categorize(rid)
|
||||
params_b = extract_params_b(rid)
|
||||
quant = "Q4_K_M" # Referenz-Quant für die Fit-Einschätzung
|
||||
fit = evaluate_fit(params_b, quant, 8192, ram_gb, name=rid)
|
||||
tags = [str(t) for t in (m.get("tags") or [])]
|
||||
by_cat[role].append({
|
||||
"name": rid.split("/")[-1], "author": rid.split("/")[0], "repo": rid,
|
||||
"role": role, "params_b": params_b, "quant": quant, "tags": tags,
|
||||
"downloads": int(m.get("downloads") or 0), "likes": int(m.get("likes") or 0),
|
||||
"lastModified": m.get("lastModified"),
|
||||
"fit": fit, "optimal_ctx": max_ctx_for(params_b, quant, ram_gb),
|
||||
"caps": capabilities(name=rid, hf={"tags": tags}),
|
||||
})
|
||||
|
||||
cats = []
|
||||
for c in CATEGORIES:
|
||||
role = c["role"]
|
||||
# 1) KATALOG zuerst (kuratierte, korrekte Metadaten, MoE-bewusst gerankt) —
|
||||
# macht die Empfehlung präzise statt Namens-Raterei.
|
||||
cat_entries = sorted(catalog.entries_for_role(role),
|
||||
key=lambda e: -catalog.stack_score(e, ram_gb))
|
||||
cat_models = [m for m in (catalog.to_model_dict(e, ram_gb) for e in cat_entries)
|
||||
if m["fit"]["level"] != "too_tight"]
|
||||
# 2) HF-Dynamik als Ergänzung (nicht-kuratierte Funde), dedupliziert.
|
||||
hf_ranked = rank_runnable(by_cat[role])
|
||||
seen = {catalog._norm(m["repo"]) for m in cat_models}
|
||||
extra = [h for h in hf_ranked if catalog._norm(h["repo"]) not in seen]
|
||||
combined = cat_models + extra
|
||||
if combined:
|
||||
cats.append({
|
||||
"role": role, "title": c["title"], "icon": c["icon"],
|
||||
"models": combined[:6],
|
||||
# Empfehlung = bester KURATIERTER Eintrag, sonst beste HF-Fundstelle.
|
||||
"recommended": (cat_models[0]["repo"] if cat_models
|
||||
else (hf_ranked[0]["repo"] if hf_ranked else None)),
|
||||
})
|
||||
|
||||
data = {"updated": time.time(), "categories": cats}
|
||||
try:
|
||||
DISCOVER_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = DISCOVER_CACHE_PATH.with_name(DISCOVER_CACHE_PATH.name + ".tmp")
|
||||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, DISCOVER_CACHE_PATH)
|
||||
except Exception:
|
||||
log.debug("discover: Cache-Schreiben fehlgeschlagen (nur Beschleunigung)", exc_info=True)
|
||||
return data
|
||||
|
||||
|
||||
def load_discover() -> dict | None:
|
||||
try:
|
||||
if DISCOVER_CACHE_PATH.exists():
|
||||
return json.loads(DISCOVER_CACHE_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
log.debug("discover: Cache-Lesen fehlgeschlagen", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def safe_discover(ram_gb: float) -> dict | None:
|
||||
"""Aus Cache (wenn frisch) oder live; wirft nie — None wenn nichts da."""
|
||||
cached = load_discover()
|
||||
if cached and (time.time() - cached.get("updated", 0) < DISCOVER_TTL):
|
||||
return cached
|
||||
try:
|
||||
return refresh_discover(ram_gb)
|
||||
except Exception:
|
||||
log.warning("discover: Live-Refresh fehlgeschlagen, nutze Cache", exc_info=True)
|
||||
return cached
|
||||
"""
|
||||
Automatische Modell-Entdeckung ("aktuell beste Modelle"): fragt vertrauenswürdige
|
||||
HF-Orgs live ab, kategorisiert per Stichwort, rankt nach Hardware-Fit + Beliebtheit
|
||||
und cached. Portiert aus Mission Control v1 (cookbook.py-Discover).
|
||||
|
||||
Wichtig (Greenfield-Fix gegen v1): EIN gemeinsamer Ranking-Helfer `rank_runnable`
|
||||
ist die Quelle der Wahrheit — sowohl die „beste Empfehlung" je Kategorie als auch
|
||||
spätere Auto-Setups nutzen ihn, damit sie nie auseinanderlaufen.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
|
||||
import logging
|
||||
|
||||
from config import DISCOVER_CACHE_PATH, DISCOVER_TTL
|
||||
from services import catalog
|
||||
from services.caps import capabilities
|
||||
from services.fit import evaluate_fit, extract_params_b, max_ctx_for
|
||||
from services.sources import CATEGORIES, SKIP_TOKENS, TRUSTED_AUTHORS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_FIT_ORDER = {"perfect": 0, "marginal": 1, "too_tight": 2}
|
||||
|
||||
|
||||
def _categorize(repo_id: str) -> str:
|
||||
low = repo_id.lower()
|
||||
for cat in CATEGORIES:
|
||||
if any(k in low for k in cat["kw"]):
|
||||
return cat["role"]
|
||||
return "scout"
|
||||
|
||||
|
||||
def _fetch_author_models(author: str) -> list:
|
||||
url = (f"https://huggingface.co/api/models?author={author}"
|
||||
f"&filter=gguf&sort=downloads&direction=-1&limit=40")
|
||||
try:
|
||||
with httpx.Client(timeout=12.0) as c:
|
||||
data = c.get(url).json()
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
log.debug("discover: Abfrage für Autor %s fehlgeschlagen", author, exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
def _age_days(last_modified, now_ts: float) -> float:
|
||||
"""Alter eines HF-Modells in Tagen (lastModified ISO). Unbekannt → ~1.5 Jahre."""
|
||||
if not last_modified:
|
||||
return 540.0
|
||||
try:
|
||||
dt = datetime.fromisoformat(str(last_modified).replace("Z", "+00:00"))
|
||||
return max((now_ts - dt.timestamp()) / 86400.0, 0.0)
|
||||
except Exception:
|
||||
return 540.0
|
||||
|
||||
|
||||
def _score(m: dict, now_ts: float) -> float:
|
||||
"""Zukunftssicherer Rang-Score für DIESE Hardware. Kombiniert:
|
||||
- Fit: perfect dominiert (Bonus 3.0 > Summe der übrigen Terme → passt-komfortabel zuerst),
|
||||
- Recency: neuere Generationen bevorzugt (Halbwertszeit ~9 Monate über lastModified),
|
||||
- Capability: mehr Parameter (log-skaliert),
|
||||
- Popularity: Downloads (log-skaliert).
|
||||
So gewinnt bei vergleichbarer Größe die NEUERE Generation (z.B. Qwen3-Coder vor
|
||||
Qwen2.5-Coder), ohne dass kleine Populär-Modelle große verdrängen."""
|
||||
fit_bonus = 3.0 if m["fit"]["level"] == "perfect" else 0.0
|
||||
recency = 0.5 ** (_age_days(m.get("lastModified"), now_ts) / 270.0)
|
||||
cap = math.log2(max(float(m.get("params_b") or 1.0), 1.0) + 1.0) / 8.0
|
||||
pop = math.log10(float(m.get("downloads") or 0) + 1.0) / 7.0
|
||||
return fit_bonus + 1.2 * recency + 1.2 * cap + 0.5 * pop
|
||||
|
||||
|
||||
def rank_runnable(models: list[dict]) -> list[dict]:
|
||||
"""EINE Quelle der Wahrheit fürs Ranking lauffähiger Modelle für DIESE Hardware.
|
||||
Nur was passt (too_tight fliegt raus), dann nach `_score` (Fit + Recency + Capability
|
||||
+ Popularity). Bevorzugt neuere, fähige Modelle → zukunftssicher; „Modelle finden"
|
||||
schlägt nie ein Downgrade vor (Downgrade-Sperre zusätzlich in maintenance)."""
|
||||
now_ts = time.time()
|
||||
return sorted(
|
||||
[m for m in models if m["fit"]["level"] != "too_tight"],
|
||||
key=lambda m: -_score(m, now_ts),
|
||||
)
|
||||
|
||||
|
||||
def refresh_discover(ram_gb: float) -> dict:
|
||||
"""Quellen live abfragen, kategorisieren, ranken, cachen. Wirft nur, wenn KEINE
|
||||
Quelle erreichbar war."""
|
||||
raw, seen, ok = [], set(), 0
|
||||
for author in TRUSTED_AUTHORS:
|
||||
models = _fetch_author_models(author)
|
||||
if models:
|
||||
ok += 1
|
||||
for m in models:
|
||||
rid = m.get("id")
|
||||
if not rid or rid in seen:
|
||||
continue
|
||||
seen.add(rid)
|
||||
raw.append(m)
|
||||
if ok == 0 and not catalog.entries():
|
||||
raise RuntimeError("Keine Quelle erreichbar.")
|
||||
|
||||
by_cat: dict[str, list] = {c["role"]: [] for c in CATEGORIES}
|
||||
for m in raw:
|
||||
rid = m["id"]
|
||||
low = rid.lower()
|
||||
if any(tok in low for tok in SKIP_TOKENS):
|
||||
continue
|
||||
role = _categorize(rid)
|
||||
params_b = extract_params_b(rid)
|
||||
quant = "Q4_K_M" # Referenz-Quant für die Fit-Einschätzung
|
||||
fit = evaluate_fit(params_b, quant, 8192, ram_gb, name=rid)
|
||||
tags = [str(t) for t in (m.get("tags") or [])]
|
||||
by_cat[role].append({
|
||||
"name": rid.split("/")[-1], "author": rid.split("/")[0], "repo": rid,
|
||||
"role": role, "params_b": params_b, "quant": quant, "tags": tags,
|
||||
"downloads": int(m.get("downloads") or 0), "likes": int(m.get("likes") or 0),
|
||||
"lastModified": m.get("lastModified"),
|
||||
"fit": fit, "optimal_ctx": max_ctx_for(params_b, quant, ram_gb),
|
||||
"caps": capabilities(name=rid, hf={"tags": tags}),
|
||||
})
|
||||
|
||||
cats = []
|
||||
for c in CATEGORIES:
|
||||
role = c["role"]
|
||||
# 1) KATALOG zuerst (kuratierte, korrekte Metadaten, MoE-bewusst gerankt) —
|
||||
# macht die Empfehlung präzise statt Namens-Raterei.
|
||||
cat_entries = sorted(catalog.entries_for_role(role),
|
||||
key=lambda e: -catalog.stack_score(e, ram_gb))
|
||||
cat_models = [m for m in (catalog.to_model_dict(e, ram_gb) for e in cat_entries)
|
||||
if m["fit"]["level"] != "too_tight"]
|
||||
# 2) HF-Dynamik als Ergänzung (nicht-kuratierte Funde), dedupliziert.
|
||||
hf_ranked = rank_runnable(by_cat[role])
|
||||
seen = {catalog._norm(m["repo"]) for m in cat_models}
|
||||
extra = [h for h in hf_ranked if catalog._norm(h["repo"]) not in seen]
|
||||
combined = cat_models + extra
|
||||
if combined:
|
||||
cats.append({
|
||||
"role": role, "title": c["title"], "icon": c["icon"],
|
||||
"models": combined[:6],
|
||||
# Empfehlung = bester KURATIERTER Eintrag, sonst beste HF-Fundstelle.
|
||||
"recommended": (cat_models[0]["repo"] if cat_models
|
||||
else (hf_ranked[0]["repo"] if hf_ranked else None)),
|
||||
})
|
||||
|
||||
data = {"updated": time.time(), "categories": cats}
|
||||
try:
|
||||
DISCOVER_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = DISCOVER_CACHE_PATH.with_name(DISCOVER_CACHE_PATH.name + ".tmp")
|
||||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, DISCOVER_CACHE_PATH)
|
||||
except Exception:
|
||||
log.debug("discover: Cache-Schreiben fehlgeschlagen (nur Beschleunigung)", exc_info=True)
|
||||
return data
|
||||
|
||||
|
||||
def load_discover() -> dict | None:
|
||||
try:
|
||||
if DISCOVER_CACHE_PATH.exists():
|
||||
return json.loads(DISCOVER_CACHE_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
log.debug("discover: Cache-Lesen fehlgeschlagen", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def safe_discover(ram_gb: float) -> dict | None:
|
||||
"""Aus Cache (wenn frisch) oder live; wirft nie — None wenn nichts da."""
|
||||
cached = load_discover()
|
||||
if cached and (time.time() - cached.get("updated", 0) < DISCOVER_TTL):
|
||||
return cached
|
||||
try:
|
||||
return refresh_discover(ram_gb)
|
||||
except Exception:
|
||||
log.warning("discover: Live-Refresh fehlgeschlagen, nutze Cache", exc_info=True)
|
||||
return cached
|
||||
|
||||
@@ -1,41 +1,41 @@
|
||||
"""Token-Erfassung für den Builtin-Gateway.
|
||||
|
||||
Parst die `usage`-Felder aus llama-swap-Antworten (Stream + Non-Stream) und meldet
|
||||
sie an token_stats. Hält den gateway_proxy-Router dünn und ersetzt die zuvor inline
|
||||
verstreute, still scheiternde String-Suche durch einen testbaren SSE-Zeilenparser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from services.token_stats import increment_tokens
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def record_usage(usage: dict | None, model: str) -> None:
|
||||
"""Ein usage-Objekt verbuchen (no-op bei None/leer)."""
|
||||
if not usage:
|
||||
return
|
||||
prompt = usage.get("prompt_tokens", 0)
|
||||
completion = usage.get("completion_tokens", 0)
|
||||
if prompt or completion:
|
||||
increment_tokens(prompt, completion, model=model)
|
||||
|
||||
|
||||
def record_stream_chunk(chunk: bytes, model: str) -> None:
|
||||
"""Rohen SSE-Chunk auf `usage` prüfen und Tokens verbuchen. Fehler werden
|
||||
geloggt (debug) statt verschluckt — ein defekter Chunk bricht den Stream nicht."""
|
||||
if b'"usage"' not in chunk:
|
||||
return
|
||||
text = chunk.decode("utf-8", errors="ignore")
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[5:].strip()
|
||||
if not data_str or data_str == "[DONE]":
|
||||
continue
|
||||
try:
|
||||
record_usage(json.loads(data_str).get("usage"), model)
|
||||
except json.JSONDecodeError:
|
||||
log.debug("gateway stream: usage-Parsing fehlgeschlagen: %s", data_str[:120])
|
||||
"""Token-Erfassung für den Builtin-Gateway.
|
||||
|
||||
Parst die `usage`-Felder aus llama-swap-Antworten (Stream + Non-Stream) und meldet
|
||||
sie an token_stats. Hält den gateway_proxy-Router dünn und ersetzt die zuvor inline
|
||||
verstreute, still scheiternde String-Suche durch einen testbaren SSE-Zeilenparser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from services.token_stats import increment_tokens
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def record_usage(usage: dict | None, model: str) -> None:
|
||||
"""Ein usage-Objekt verbuchen (no-op bei None/leer)."""
|
||||
if not usage:
|
||||
return
|
||||
prompt = usage.get("prompt_tokens", 0)
|
||||
completion = usage.get("completion_tokens", 0)
|
||||
if prompt or completion:
|
||||
increment_tokens(prompt, completion, model=model)
|
||||
|
||||
|
||||
def record_stream_chunk(chunk: bytes, model: str) -> None:
|
||||
"""Rohen SSE-Chunk auf `usage` prüfen und Tokens verbuchen. Fehler werden
|
||||
geloggt (debug) statt verschluckt — ein defekter Chunk bricht den Stream nicht."""
|
||||
if b'"usage"' not in chunk:
|
||||
return
|
||||
text = chunk.decode("utf-8", errors="ignore")
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[5:].strip()
|
||||
if not data_str or data_str == "[DONE]":
|
||||
continue
|
||||
try:
|
||||
record_usage(json.loads(data_str).get("usage"), model)
|
||||
except json.JSONDecodeError:
|
||||
log.debug("gateway stream: usage-Parsing fehlgeschlagen: %s", data_str[:120])
|
||||
|
||||
+266
-266
@@ -1,266 +1,266 @@
|
||||
"""
|
||||
GGUF-Tokenizer-Fingerprint — liest die Tokenizer-Identität direkt aus dem
|
||||
GGUF-Header (ohne das Modell zu laden), um zu entscheiden, ob ein Draft-Modell
|
||||
**vocab-kompatibel** mit einem Ziel-Modell ist (Voraussetzung für Speculative
|
||||
Decoding in llama.cpp — sonst: "draft model vocab type must match target").
|
||||
|
||||
Wir lesen nur die Metadaten-KV-Sektion am Dateianfang und brechen ab, sobald
|
||||
`tokenizer.ggml.tokens` erreicht ist (dessen Länge = n_vocab). model+pre+n_vocab
|
||||
identifizieren den Tokenizer eindeutig genug, um die in der Praxis relevanten
|
||||
Fälle zu unterscheiden (Qwen2.5 vs Qwen3 vs Qwen3.6 etc.). Die llama.cpp-Prüfung
|
||||
beim Laden bleibt der letzte Schiedsrichter.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import struct
|
||||
from functools import lru_cache
|
||||
|
||||
# GGUF value types (https://github.com/ggml-org/ggml/blob/master/docs/gguf.md)
|
||||
_T_UINT8, _T_INT8, _T_UINT16, _T_INT16, _T_UINT32, _T_INT32, _T_FLOAT32, \
|
||||
_T_BOOL, _T_STRING, _T_ARRAY, _T_UINT64, _T_INT64, _T_FLOAT64 = range(13)
|
||||
|
||||
_SCALAR_FMT = {
|
||||
_T_UINT8: "<B", _T_INT8: "<b", _T_UINT16: "<H", _T_INT16: "<h",
|
||||
_T_UINT32: "<I", _T_INT32: "<i", _T_FLOAT32: "<f", _T_BOOL: "<?",
|
||||
_T_UINT64: "<Q", _T_INT64: "<q", _T_FLOAT64: "<d",
|
||||
}
|
||||
_SCALAR_SIZE = {t: struct.calcsize(f) for t, f in _SCALAR_FMT.items()}
|
||||
|
||||
_WANT_STRINGS = {"tokenizer.ggml.model", "tokenizer.ggml.pre", "general.architecture"}
|
||||
|
||||
|
||||
class _Reader:
|
||||
def __init__(self, f):
|
||||
self.f = f
|
||||
|
||||
def read(self, n: int) -> bytes:
|
||||
b = self.f.read(n)
|
||||
if len(b) != n:
|
||||
raise EOFError("unerwartetes Dateiende beim GGUF-Parsen")
|
||||
return b
|
||||
|
||||
def u32(self) -> int:
|
||||
return struct.unpack("<I", self.read(4))[0]
|
||||
|
||||
def u64(self) -> int:
|
||||
return struct.unpack("<Q", self.read(8))[0]
|
||||
|
||||
def gstr(self) -> str:
|
||||
n = self.u64()
|
||||
return self.read(n).decode("utf-8", "replace")
|
||||
|
||||
def scalar(self, vtype: int):
|
||||
"""Liest einen Skalar-Wert (für die Architektur-Metadaten). None bei Nicht-Skalar."""
|
||||
fmt = _SCALAR_FMT.get(vtype)
|
||||
if not fmt:
|
||||
self.skip_value(vtype)
|
||||
return None
|
||||
return struct.unpack(fmt, self.read(_SCALAR_SIZE[vtype]))[0]
|
||||
|
||||
def skip_value(self, vtype: int) -> None:
|
||||
"""Liest einen Wert und verwirft ihn (um den Datei-Pointer korrekt
|
||||
weiterzuschieben). Arrays werden elementweise konsumiert."""
|
||||
if vtype == _T_STRING:
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif vtype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[vtype], 1)
|
||||
elif vtype == _T_ARRAY:
|
||||
etype = self.u32()
|
||||
count = self.u64()
|
||||
if etype == _T_STRING:
|
||||
for _ in range(count):
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif etype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[etype] * count, 1)
|
||||
else:
|
||||
raise ValueError(f"unbekannter Array-Elementtyp {etype}")
|
||||
else:
|
||||
raise ValueError(f"unbekannter GGUF-Wertetyp {vtype}")
|
||||
|
||||
|
||||
def _read_fingerprint(path: str) -> dict | None:
|
||||
"""Liest model/pre/n_vocab aus dem GGUF-Header. None bei Fehler/kein GGUF."""
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
r = _Reader(fh)
|
||||
if r.read(4) != b"GGUF":
|
||||
return None
|
||||
r.u32() # version
|
||||
r.u64() # tensor_count
|
||||
kv_count = r.u64()
|
||||
fp: dict = {"model": None, "pre": None, "arch": None, "n_vocab": None,
|
||||
"tokens_sha": None}
|
||||
for _ in range(kv_count):
|
||||
key = r.gstr()
|
||||
vtype = r.u32()
|
||||
if key == "tokenizer.ggml.tokens" and vtype == _T_ARRAY:
|
||||
etype = r.u32()
|
||||
count = r.u64()
|
||||
fp["n_vocab"] = count
|
||||
if etype != _T_STRING:
|
||||
return None
|
||||
# ECHTE Vocab-Identität: sha256 über die tatsächliche Token-Liste
|
||||
# (familienunabhängig — funktioniert für Qwen, Llama, Mistral, …).
|
||||
h = hashlib.sha256()
|
||||
h.update(count.to_bytes(8, "little"))
|
||||
for _ in range(count):
|
||||
n = r.u64()
|
||||
h.update(r.read(n))
|
||||
fp["tokens_sha"] = h.hexdigest()
|
||||
# model/pre kommen vor tokens → wir haben alles. Abbrechen.
|
||||
break
|
||||
if key in _WANT_STRINGS and vtype == _T_STRING:
|
||||
val = r.gstr()
|
||||
if key == "tokenizer.ggml.model":
|
||||
fp["model"] = val
|
||||
elif key == "tokenizer.ggml.pre":
|
||||
fp["pre"] = val
|
||||
else:
|
||||
fp["arch"] = val
|
||||
else:
|
||||
r.skip_value(vtype)
|
||||
if fp["model"] is None and fp["n_vocab"] is None:
|
||||
return None
|
||||
return fp
|
||||
except (OSError, EOFError, ValueError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _cached(path: str, mtime: float, size: int) -> tuple | None:
|
||||
fp = _read_fingerprint(path)
|
||||
if fp is None:
|
||||
return None
|
||||
return (fp.get("model"), fp.get("pre"), fp.get("n_vocab"), fp.get("arch"), fp.get("tokens_sha"))
|
||||
|
||||
|
||||
def fingerprint(path: str) -> dict | None:
|
||||
"""Tokenizer-Fingerprint eines GGUF (gecacht nach Pfad+mtime+size).
|
||||
Returns dict(model, pre, n_vocab, arch, tokens_sha) oder None wenn nicht lesbar."""
|
||||
import os
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
t = _cached(path, st.st_mtime, st.st_size)
|
||||
if t is None:
|
||||
return None
|
||||
return {"model": t[0], "pre": t[1], "n_vocab": t[2], "arch": t[3], "tokens_sha": t[4]}
|
||||
|
||||
|
||||
def vocab_key(path: str) -> tuple | None:
|
||||
"""ECHTER Vergleichsschlüssel für Vocab-Kompatibilität: (model, pre, n_vocab, sha256
|
||||
der vollständigen Token-Liste). Vergleicht den TATSÄCHLICHEN Vokabular-Inhalt, nicht
|
||||
nur Metadaten — familienunabhängig (Qwen, Llama, Mistral, …). Genau diese Identität
|
||||
verlangt llama.cpp für Speculative Decoding."""
|
||||
fp = fingerprint(path)
|
||||
if not fp or fp["n_vocab"] is None or not fp.get("tokens_sha"):
|
||||
return None
|
||||
return (fp["model"], fp["pre"], fp["n_vocab"], fp["tokens_sha"])
|
||||
|
||||
|
||||
def compatible(target_path: str, draft_path: str) -> bool | None:
|
||||
"""True/False ob draft vocab-kompatibel zum target ist. None = unbestimmbar
|
||||
(eine Datei nicht lesbar) → UI behandelt das als 'nicht bestätigt'."""
|
||||
a = vocab_key(target_path)
|
||||
b = vocab_key(draft_path)
|
||||
if a is None or b is None:
|
||||
return None
|
||||
return a == b
|
||||
|
||||
|
||||
# ── Architektur-Metadaten für EHRLICHE KV-Cache-Größen ──────────────────────────────
|
||||
# Der KV-Cache hängt an (Layer × KV-Heads × Head-Dim), NICHT an den Gesamt-Parametern.
|
||||
# Bei MoE (z.B. Qwen3.6-35B-A3B) ist das entscheidend: die alte params-basierte Schätzung
|
||||
# überschätzte grob (aktive vs. gesamte Params + GQA), reale KV liest man direkt hier.
|
||||
# Schlüssel sind arch-präfixiert ('qwen3moe.block_count', 'llama.attention.head_count_kv' …),
|
||||
# gegen echte GGUFs verifiziert. Wir sammeln die gewünschten Skalar-Schlüssel per Suffix.
|
||||
_ARCH_WANT = (
|
||||
".block_count", ".attention.head_count_kv", ".attention.head_count",
|
||||
".attention.key_length", ".attention.value_length", ".embedding_length",
|
||||
".context_length",
|
||||
)
|
||||
|
||||
|
||||
def _read_arch_meta(path: str) -> dict | None:
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
r = _Reader(fh)
|
||||
if r.read(4) != b"GGUF":
|
||||
return None
|
||||
r.u32() # version
|
||||
r.u64() # tensor_count
|
||||
kv_count = r.u64()
|
||||
raw: dict = {}
|
||||
arch = None
|
||||
for _ in range(kv_count):
|
||||
key = r.gstr()
|
||||
vtype = r.u32()
|
||||
if key == "general.architecture" and vtype == _T_STRING:
|
||||
arch = r.gstr()
|
||||
continue
|
||||
if key == "tokenizer.ggml.tokens":
|
||||
break # Arch-Metadaten stehen davor → fertig, Rest überspringen
|
||||
suf = next((s for s in _ARCH_WANT if key.endswith(s)), None)
|
||||
if suf is not None and vtype in _SCALAR_SIZE:
|
||||
raw[suf] = r.scalar(vtype)
|
||||
else:
|
||||
r.skip_value(vtype)
|
||||
n_layers = raw.get(".block_count")
|
||||
n_head = raw.get(".attention.head_count")
|
||||
n_head_kv = raw.get(".attention.head_count_kv") or n_head # GQA fehlt → MHA
|
||||
n_embd = raw.get(".embedding_length")
|
||||
hd_k = raw.get(".attention.key_length") \
|
||||
or (int(n_embd / n_head) if (n_embd and n_head) else None)
|
||||
hd_v = raw.get(".attention.value_length") or hd_k
|
||||
if not (n_layers and n_head_kv and hd_k and hd_v):
|
||||
return None # unvollständig → Aufrufer nutzt Heuristik-Fallback
|
||||
return {"arch": arch, "n_layers": int(n_layers), "n_head_kv": int(n_head_kv),
|
||||
"head_dim_k": int(hd_k), "head_dim_v": int(hd_v),
|
||||
"n_ctx_train": int(raw[".context_length"]) if raw.get(".context_length") else None}
|
||||
except (OSError, EOFError, ValueError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _arch_cached(path: str, mtime: float, size: int) -> dict | None:
|
||||
return _read_arch_meta(path)
|
||||
|
||||
|
||||
def arch_meta(path: str) -> dict | None:
|
||||
"""Architektur-Metadaten eines GGUF (gecacht nach Pfad+mtime+size):
|
||||
{arch, n_layers, n_head_kv, head_dim_k, head_dim_v, n_ctx_train}. None wenn nicht lesbar
|
||||
oder unvollständig."""
|
||||
import os
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
return _arch_cached(path, st.st_mtime, st.st_size)
|
||||
|
||||
|
||||
# Bytes pro KV-Cache-Element je cache-type (inkl. Block-Overhead der k-Quants).
|
||||
_KV_BPE = {
|
||||
"f32": 4.0, "f16": 2.0, "bf16": 2.0,
|
||||
"q8_0": 1.0625, "q5_1": 0.75, "q5_0": 0.6875,
|
||||
"q4_1": 0.625, "q4_0": 0.5625, "iq4_nl": 0.5625,
|
||||
}
|
||||
_GIB = 1024 ** 3
|
||||
|
||||
|
||||
def _bpe(cache_type: str | None) -> float:
|
||||
return _KV_BPE.get((cache_type or "f16").lower(), 2.0)
|
||||
|
||||
|
||||
def kv_cache_gb(meta: dict, ctx: int, ck: str | None = None, cv: str | None = None) -> float:
|
||||
"""Echte KV-Cache-Größe (GiB) für ctx Tokens, K/V ggf. quantisiert. Formel wie llama.cpp:
|
||||
je Layer & Token hält der Cache n_head_kv × head_dim Elemente für K und für V."""
|
||||
per_tok = meta["n_layers"] * meta["n_head_kv"] * ctx
|
||||
k = per_tok * meta["head_dim_k"] * _bpe(ck)
|
||||
v = per_tok * meta["head_dim_v"] * _bpe(cv)
|
||||
return (k + v) / _GIB
|
||||
|
||||
|
||||
def kv_gb_per_token(meta: dict, ck: str | None = None, cv: str | None = None) -> float:
|
||||
"""KV-GiB pro Kontext-Token — für den analytischen ctx-Solver (linear in ctx)."""
|
||||
return kv_cache_gb(meta, 1, ck, cv)
|
||||
"""
|
||||
GGUF-Tokenizer-Fingerprint — liest die Tokenizer-Identität direkt aus dem
|
||||
GGUF-Header (ohne das Modell zu laden), um zu entscheiden, ob ein Draft-Modell
|
||||
**vocab-kompatibel** mit einem Ziel-Modell ist (Voraussetzung für Speculative
|
||||
Decoding in llama.cpp — sonst: "draft model vocab type must match target").
|
||||
|
||||
Wir lesen nur die Metadaten-KV-Sektion am Dateianfang und brechen ab, sobald
|
||||
`tokenizer.ggml.tokens` erreicht ist (dessen Länge = n_vocab). model+pre+n_vocab
|
||||
identifizieren den Tokenizer eindeutig genug, um die in der Praxis relevanten
|
||||
Fälle zu unterscheiden (Qwen2.5 vs Qwen3 vs Qwen3.6 etc.). Die llama.cpp-Prüfung
|
||||
beim Laden bleibt der letzte Schiedsrichter.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import struct
|
||||
from functools import lru_cache
|
||||
|
||||
# GGUF value types (https://github.com/ggml-org/ggml/blob/master/docs/gguf.md)
|
||||
_T_UINT8, _T_INT8, _T_UINT16, _T_INT16, _T_UINT32, _T_INT32, _T_FLOAT32, \
|
||||
_T_BOOL, _T_STRING, _T_ARRAY, _T_UINT64, _T_INT64, _T_FLOAT64 = range(13)
|
||||
|
||||
_SCALAR_FMT = {
|
||||
_T_UINT8: "<B", _T_INT8: "<b", _T_UINT16: "<H", _T_INT16: "<h",
|
||||
_T_UINT32: "<I", _T_INT32: "<i", _T_FLOAT32: "<f", _T_BOOL: "<?",
|
||||
_T_UINT64: "<Q", _T_INT64: "<q", _T_FLOAT64: "<d",
|
||||
}
|
||||
_SCALAR_SIZE = {t: struct.calcsize(f) for t, f in _SCALAR_FMT.items()}
|
||||
|
||||
_WANT_STRINGS = {"tokenizer.ggml.model", "tokenizer.ggml.pre", "general.architecture"}
|
||||
|
||||
|
||||
class _Reader:
|
||||
def __init__(self, f):
|
||||
self.f = f
|
||||
|
||||
def read(self, n: int) -> bytes:
|
||||
b = self.f.read(n)
|
||||
if len(b) != n:
|
||||
raise EOFError("unerwartetes Dateiende beim GGUF-Parsen")
|
||||
return b
|
||||
|
||||
def u32(self) -> int:
|
||||
return struct.unpack("<I", self.read(4))[0]
|
||||
|
||||
def u64(self) -> int:
|
||||
return struct.unpack("<Q", self.read(8))[0]
|
||||
|
||||
def gstr(self) -> str:
|
||||
n = self.u64()
|
||||
return self.read(n).decode("utf-8", "replace")
|
||||
|
||||
def scalar(self, vtype: int):
|
||||
"""Liest einen Skalar-Wert (für die Architektur-Metadaten). None bei Nicht-Skalar."""
|
||||
fmt = _SCALAR_FMT.get(vtype)
|
||||
if not fmt:
|
||||
self.skip_value(vtype)
|
||||
return None
|
||||
return struct.unpack(fmt, self.read(_SCALAR_SIZE[vtype]))[0]
|
||||
|
||||
def skip_value(self, vtype: int) -> None:
|
||||
"""Liest einen Wert und verwirft ihn (um den Datei-Pointer korrekt
|
||||
weiterzuschieben). Arrays werden elementweise konsumiert."""
|
||||
if vtype == _T_STRING:
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif vtype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[vtype], 1)
|
||||
elif vtype == _T_ARRAY:
|
||||
etype = self.u32()
|
||||
count = self.u64()
|
||||
if etype == _T_STRING:
|
||||
for _ in range(count):
|
||||
self.f.seek(self.u64(), 1)
|
||||
elif etype in _SCALAR_SIZE:
|
||||
self.f.seek(_SCALAR_SIZE[etype] * count, 1)
|
||||
else:
|
||||
raise ValueError(f"unbekannter Array-Elementtyp {etype}")
|
||||
else:
|
||||
raise ValueError(f"unbekannter GGUF-Wertetyp {vtype}")
|
||||
|
||||
|
||||
def _read_fingerprint(path: str) -> dict | None:
|
||||
"""Liest model/pre/n_vocab aus dem GGUF-Header. None bei Fehler/kein GGUF."""
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
r = _Reader(fh)
|
||||
if r.read(4) != b"GGUF":
|
||||
return None
|
||||
r.u32() # version
|
||||
r.u64() # tensor_count
|
||||
kv_count = r.u64()
|
||||
fp: dict = {"model": None, "pre": None, "arch": None, "n_vocab": None,
|
||||
"tokens_sha": None}
|
||||
for _ in range(kv_count):
|
||||
key = r.gstr()
|
||||
vtype = r.u32()
|
||||
if key == "tokenizer.ggml.tokens" and vtype == _T_ARRAY:
|
||||
etype = r.u32()
|
||||
count = r.u64()
|
||||
fp["n_vocab"] = count
|
||||
if etype != _T_STRING:
|
||||
return None
|
||||
# ECHTE Vocab-Identität: sha256 über die tatsächliche Token-Liste
|
||||
# (familienunabhängig — funktioniert für Qwen, Llama, Mistral, …).
|
||||
h = hashlib.sha256()
|
||||
h.update(count.to_bytes(8, "little"))
|
||||
for _ in range(count):
|
||||
n = r.u64()
|
||||
h.update(r.read(n))
|
||||
fp["tokens_sha"] = h.hexdigest()
|
||||
# model/pre kommen vor tokens → wir haben alles. Abbrechen.
|
||||
break
|
||||
if key in _WANT_STRINGS and vtype == _T_STRING:
|
||||
val = r.gstr()
|
||||
if key == "tokenizer.ggml.model":
|
||||
fp["model"] = val
|
||||
elif key == "tokenizer.ggml.pre":
|
||||
fp["pre"] = val
|
||||
else:
|
||||
fp["arch"] = val
|
||||
else:
|
||||
r.skip_value(vtype)
|
||||
if fp["model"] is None and fp["n_vocab"] is None:
|
||||
return None
|
||||
return fp
|
||||
except (OSError, EOFError, ValueError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _cached(path: str, mtime: float, size: int) -> tuple | None:
|
||||
fp = _read_fingerprint(path)
|
||||
if fp is None:
|
||||
return None
|
||||
return (fp.get("model"), fp.get("pre"), fp.get("n_vocab"), fp.get("arch"), fp.get("tokens_sha"))
|
||||
|
||||
|
||||
def fingerprint(path: str) -> dict | None:
|
||||
"""Tokenizer-Fingerprint eines GGUF (gecacht nach Pfad+mtime+size).
|
||||
Returns dict(model, pre, n_vocab, arch, tokens_sha) oder None wenn nicht lesbar."""
|
||||
import os
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
t = _cached(path, st.st_mtime, st.st_size)
|
||||
if t is None:
|
||||
return None
|
||||
return {"model": t[0], "pre": t[1], "n_vocab": t[2], "arch": t[3], "tokens_sha": t[4]}
|
||||
|
||||
|
||||
def vocab_key(path: str) -> tuple | None:
|
||||
"""ECHTER Vergleichsschlüssel für Vocab-Kompatibilität: (model, pre, n_vocab, sha256
|
||||
der vollständigen Token-Liste). Vergleicht den TATSÄCHLICHEN Vokabular-Inhalt, nicht
|
||||
nur Metadaten — familienunabhängig (Qwen, Llama, Mistral, …). Genau diese Identität
|
||||
verlangt llama.cpp für Speculative Decoding."""
|
||||
fp = fingerprint(path)
|
||||
if not fp or fp["n_vocab"] is None or not fp.get("tokens_sha"):
|
||||
return None
|
||||
return (fp["model"], fp["pre"], fp["n_vocab"], fp["tokens_sha"])
|
||||
|
||||
|
||||
def compatible(target_path: str, draft_path: str) -> bool | None:
|
||||
"""True/False ob draft vocab-kompatibel zum target ist. None = unbestimmbar
|
||||
(eine Datei nicht lesbar) → UI behandelt das als 'nicht bestätigt'."""
|
||||
a = vocab_key(target_path)
|
||||
b = vocab_key(draft_path)
|
||||
if a is None or b is None:
|
||||
return None
|
||||
return a == b
|
||||
|
||||
|
||||
# ── Architektur-Metadaten für EHRLICHE KV-Cache-Größen ──────────────────────────────
|
||||
# Der KV-Cache hängt an (Layer × KV-Heads × Head-Dim), NICHT an den Gesamt-Parametern.
|
||||
# Bei MoE (z.B. Qwen3.6-35B-A3B) ist das entscheidend: die alte params-basierte Schätzung
|
||||
# überschätzte grob (aktive vs. gesamte Params + GQA), reale KV liest man direkt hier.
|
||||
# Schlüssel sind arch-präfixiert ('qwen3moe.block_count', 'llama.attention.head_count_kv' …),
|
||||
# gegen echte GGUFs verifiziert. Wir sammeln die gewünschten Skalar-Schlüssel per Suffix.
|
||||
_ARCH_WANT = (
|
||||
".block_count", ".attention.head_count_kv", ".attention.head_count",
|
||||
".attention.key_length", ".attention.value_length", ".embedding_length",
|
||||
".context_length",
|
||||
)
|
||||
|
||||
|
||||
def _read_arch_meta(path: str) -> dict | None:
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
r = _Reader(fh)
|
||||
if r.read(4) != b"GGUF":
|
||||
return None
|
||||
r.u32() # version
|
||||
r.u64() # tensor_count
|
||||
kv_count = r.u64()
|
||||
raw: dict = {}
|
||||
arch = None
|
||||
for _ in range(kv_count):
|
||||
key = r.gstr()
|
||||
vtype = r.u32()
|
||||
if key == "general.architecture" and vtype == _T_STRING:
|
||||
arch = r.gstr()
|
||||
continue
|
||||
if key == "tokenizer.ggml.tokens":
|
||||
break # Arch-Metadaten stehen davor → fertig, Rest überspringen
|
||||
suf = next((s for s in _ARCH_WANT if key.endswith(s)), None)
|
||||
if suf is not None and vtype in _SCALAR_SIZE:
|
||||
raw[suf] = r.scalar(vtype)
|
||||
else:
|
||||
r.skip_value(vtype)
|
||||
n_layers = raw.get(".block_count")
|
||||
n_head = raw.get(".attention.head_count")
|
||||
n_head_kv = raw.get(".attention.head_count_kv") or n_head # GQA fehlt → MHA
|
||||
n_embd = raw.get(".embedding_length")
|
||||
hd_k = raw.get(".attention.key_length") \
|
||||
or (int(n_embd / n_head) if (n_embd and n_head) else None)
|
||||
hd_v = raw.get(".attention.value_length") or hd_k
|
||||
if not (n_layers and n_head_kv and hd_k and hd_v):
|
||||
return None # unvollständig → Aufrufer nutzt Heuristik-Fallback
|
||||
return {"arch": arch, "n_layers": int(n_layers), "n_head_kv": int(n_head_kv),
|
||||
"head_dim_k": int(hd_k), "head_dim_v": int(hd_v),
|
||||
"n_ctx_train": int(raw[".context_length"]) if raw.get(".context_length") else None}
|
||||
except (OSError, EOFError, ValueError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _arch_cached(path: str, mtime: float, size: int) -> dict | None:
|
||||
return _read_arch_meta(path)
|
||||
|
||||
|
||||
def arch_meta(path: str) -> dict | None:
|
||||
"""Architektur-Metadaten eines GGUF (gecacht nach Pfad+mtime+size):
|
||||
{arch, n_layers, n_head_kv, head_dim_k, head_dim_v, n_ctx_train}. None wenn nicht lesbar
|
||||
oder unvollständig."""
|
||||
import os
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
return _arch_cached(path, st.st_mtime, st.st_size)
|
||||
|
||||
|
||||
# Bytes pro KV-Cache-Element je cache-type (inkl. Block-Overhead der k-Quants).
|
||||
_KV_BPE = {
|
||||
"f32": 4.0, "f16": 2.0, "bf16": 2.0,
|
||||
"q8_0": 1.0625, "q5_1": 0.75, "q5_0": 0.6875,
|
||||
"q4_1": 0.625, "q4_0": 0.5625, "iq4_nl": 0.5625,
|
||||
}
|
||||
_GIB = 1024 ** 3
|
||||
|
||||
|
||||
def _bpe(cache_type: str | None) -> float:
|
||||
return _KV_BPE.get((cache_type or "f16").lower(), 2.0)
|
||||
|
||||
|
||||
def kv_cache_gb(meta: dict, ctx: int, ck: str | None = None, cv: str | None = None) -> float:
|
||||
"""Echte KV-Cache-Größe (GiB) für ctx Tokens, K/V ggf. quantisiert. Formel wie llama.cpp:
|
||||
je Layer & Token hält der Cache n_head_kv × head_dim Elemente für K und für V."""
|
||||
per_tok = meta["n_layers"] * meta["n_head_kv"] * ctx
|
||||
k = per_tok * meta["head_dim_k"] * _bpe(ck)
|
||||
v = per_tok * meta["head_dim_v"] * _bpe(cv)
|
||||
return (k + v) / _GIB
|
||||
|
||||
|
||||
def kv_gb_per_token(meta: dict, ck: str | None = None, cv: str | None = None) -> float:
|
||||
"""KV-GiB pro Kontext-Token — für den analytischen ctx-Solver (linear in ctx)."""
|
||||
return kv_cache_gb(meta, 1, ck, cv)
|
||||
|
||||
+576
-576
File diff suppressed because it is too large
Load Diff
+726
-726
File diff suppressed because it is too large
Load Diff
@@ -161,7 +161,7 @@ def dedupe(apply: bool = False, threshold: float = 0.85) -> dict:
|
||||
AUTO_DEDUPE_ENABLED = os.environ.get("MC_MEM_DEDUPE_ENABLED", "1") != "0"
|
||||
AUTO_DEDUPE_INTERVAL = int(os.environ.get("MC_MEM_DEDUPE_INTERVAL", str(24 * 3600))) # täglich
|
||||
AUTO_DEDUPE_START_DELAY = int(os.environ.get("MC_MEM_DEDUPE_START_DELAY", "300")) # 5 min nach Start
|
||||
AUTO_DEDUPE_THRESHOLD = float(os.environ.get("MC_MEM_DEDUPE_THRESHOLD", "0.9"))
|
||||
AUTO_DEDUPE_THRESHOLD = float(os.environ.get("MC_MEM_DEDUPE_THRESHOLD", "0.75"))
|
||||
|
||||
|
||||
async def auto_dedupe_loop() -> None:
|
||||
|
||||
+58
-58
@@ -1,58 +1,58 @@
|
||||
"""Kosten-/Ersparnis-Berechnung für die Token-Statistik (eine Quelle der Wahrheit).
|
||||
|
||||
Vergleicht die lokal verbrauchten Tokens gegen die Cloud-Listenpreise vergleichbarer
|
||||
Modellklassen (Stand Juni 2026, USD pro 1M Tokens, in/out) und liefert die so
|
||||
eingesparte Summe. Wird vom System-Router dünn aufgerufen.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# Cloud-Listenpreise je Rolle/Modellklasse: (input_usd_per_1M, output_usd_per_1M).
|
||||
PRICING: dict[str, tuple[float, float]] = {
|
||||
"heavy": (15.0, 75.0),
|
||||
"coder": (3.0, 15.0),
|
||||
"hermes": (1.0, 5.0),
|
||||
"fast": (0.15, 0.60),
|
||||
"scout": (0.15, 0.60),
|
||||
"vision": (0.15, 0.60),
|
||||
"reasoning": (0.15, 0.60),
|
||||
}
|
||||
# Tarif für nicht zuordenbare Tokens (Default-/Fallback-Klasse).
|
||||
DEFAULT_RATE: tuple[float, float] = (0.15, 0.60)
|
||||
# Baseline/Legacy-Tokens (vor modellspezifischem Logging) am Premium-Tarif bewerten,
|
||||
# damit historische Ersparnis erhalten bleibt.
|
||||
BASELINE_RATE: tuple[float, float] = PRICING["heavy"]
|
||||
USD_TO_EUR = float(os.environ.get("MC_USD_TO_EUR", "0.92"))
|
||||
|
||||
|
||||
def compute_savings(stats: dict, role_map: dict[str, str | None]) -> dict:
|
||||
"""Aggregiert Tokens und berechnet die Cloud-Ersparnis.
|
||||
|
||||
role_map: Modell-/Alias-Name (lowercase) -> Rolle, zur Tarif-Auflösung.
|
||||
"""
|
||||
prompt = stats.get("prompt_tokens", 0)
|
||||
completion = stats.get("completion_tokens", 0)
|
||||
|
||||
modeled_p = modeled_c = 0
|
||||
saved_usd = 0.0
|
||||
for m_name, m_tokens in (stats.get("models") or {}).items():
|
||||
mp = m_tokens.get("prompt", 0)
|
||||
mc = m_tokens.get("completion", 0)
|
||||
modeled_p += mp
|
||||
modeled_c += mc
|
||||
role = role_map.get(m_name, m_name)
|
||||
rate_in, rate_out = PRICING.get(role, DEFAULT_RATE)
|
||||
saved_usd += (mp * rate_in + mc * rate_out) / 1_000_000.0
|
||||
|
||||
baseline_p = max(0, prompt - modeled_p)
|
||||
baseline_c = max(0, completion - modeled_c)
|
||||
saved_usd += (baseline_p * BASELINE_RATE[0] + baseline_c * BASELINE_RATE[1]) / 1_000_000.0
|
||||
|
||||
return {
|
||||
"prompt_tokens": prompt,
|
||||
"completion_tokens": completion,
|
||||
"total_tokens": prompt + completion,
|
||||
"saved_usd": round(saved_usd, 2),
|
||||
"saved_eur": round(saved_usd * USD_TO_EUR, 2),
|
||||
"pricing": {role: {"in": r[0], "out": r[1]} for role, r in PRICING.items()},
|
||||
}
|
||||
"""Kosten-/Ersparnis-Berechnung für die Token-Statistik (eine Quelle der Wahrheit).
|
||||
|
||||
Vergleicht die lokal verbrauchten Tokens gegen die Cloud-Listenpreise vergleichbarer
|
||||
Modellklassen (Stand Juni 2026, USD pro 1M Tokens, in/out) und liefert die so
|
||||
eingesparte Summe. Wird vom System-Router dünn aufgerufen.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# Cloud-Listenpreise je Rolle/Modellklasse: (input_usd_per_1M, output_usd_per_1M).
|
||||
PRICING: dict[str, tuple[float, float]] = {
|
||||
"heavy": (15.0, 75.0),
|
||||
"coder": (3.0, 15.0),
|
||||
"hermes": (1.0, 5.0),
|
||||
"fast": (0.15, 0.60),
|
||||
"scout": (0.15, 0.60),
|
||||
"vision": (0.15, 0.60),
|
||||
"reasoning": (0.15, 0.60),
|
||||
}
|
||||
# Tarif für nicht zuordenbare Tokens (Default-/Fallback-Klasse).
|
||||
DEFAULT_RATE: tuple[float, float] = (0.15, 0.60)
|
||||
# Baseline/Legacy-Tokens (vor modellspezifischem Logging) am Premium-Tarif bewerten,
|
||||
# damit historische Ersparnis erhalten bleibt.
|
||||
BASELINE_RATE: tuple[float, float] = PRICING["heavy"]
|
||||
USD_TO_EUR = float(os.environ.get("MC_USD_TO_EUR", "0.92"))
|
||||
|
||||
|
||||
def compute_savings(stats: dict, role_map: dict[str, str | None]) -> dict:
|
||||
"""Aggregiert Tokens und berechnet die Cloud-Ersparnis.
|
||||
|
||||
role_map: Modell-/Alias-Name (lowercase) -> Rolle, zur Tarif-Auflösung.
|
||||
"""
|
||||
prompt = stats.get("prompt_tokens", 0)
|
||||
completion = stats.get("completion_tokens", 0)
|
||||
|
||||
modeled_p = modeled_c = 0
|
||||
saved_usd = 0.0
|
||||
for m_name, m_tokens in (stats.get("models") or {}).items():
|
||||
mp = m_tokens.get("prompt", 0)
|
||||
mc = m_tokens.get("completion", 0)
|
||||
modeled_p += mp
|
||||
modeled_c += mc
|
||||
role = role_map.get(m_name, m_name)
|
||||
rate_in, rate_out = PRICING.get(role, DEFAULT_RATE)
|
||||
saved_usd += (mp * rate_in + mc * rate_out) / 1_000_000.0
|
||||
|
||||
baseline_p = max(0, prompt - modeled_p)
|
||||
baseline_c = max(0, completion - modeled_c)
|
||||
saved_usd += (baseline_p * BASELINE_RATE[0] + baseline_c * BASELINE_RATE[1]) / 1_000_000.0
|
||||
|
||||
return {
|
||||
"prompt_tokens": prompt,
|
||||
"completion_tokens": completion,
|
||||
"total_tokens": prompt + completion,
|
||||
"saved_usd": round(saved_usd, 2),
|
||||
"saved_eur": round(saved_usd * USD_TO_EUR, 2),
|
||||
"pricing": {role: {"in": r[0], "out": r[1]} for role, r in PRICING.items()},
|
||||
}
|
||||
|
||||
+127
-127
@@ -1,127 +1,127 @@
|
||||
"""
|
||||
UI-editierbare Routing-Policy für die Gateway-Lanes (coding/chat).
|
||||
|
||||
Persistiert als JSON unter MC_ROUTING_POLICY_PATH (Default MODELS_DIR/mc2-routing.json —
|
||||
gleiche Konvention wie mc2-discover.json). **Hot-reload:** load_policy() liest die Datei nur
|
||||
bei Änderung neu (mtime-Cache) → UI-Edits greifen ohne Dienst-Neustart. Die Env-Vars (bisher
|
||||
einzige Stellschraube in router_logic.py) bleiben als Defaults/Fallback erhalten.
|
||||
|
||||
Bewusst NICHT editierbar (v1): die Regex-Keyword-Listen (heavy/coding-heavy/code-hint) — die
|
||||
bleiben in router_logic.py im Code.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from config import MODELS_DIR
|
||||
|
||||
POLICY_PATH = Path(os.environ.get("MC_ROUTING_POLICY_PATH", str(MODELS_DIR / "mc2-routing.json")))
|
||||
|
||||
|
||||
def _env_bool(name: str, default: str) -> bool:
|
||||
return os.environ.get(name, default) not in ("0", "false", "")
|
||||
|
||||
|
||||
# Defaults aus den Env-Vars — Quelle der Wahrheit, solange keine Policy-Datei existiert.
|
||||
DEFAULTS: dict = {
|
||||
"fast": os.environ.get("MC_ROUTE_FAST", "fast"),
|
||||
"heavy": os.environ.get("MC_ROUTE_HEAVY", "heavy"),
|
||||
"coder": os.environ.get("MC_ROUTE_CODER", "coder"),
|
||||
"coder_lite": os.environ.get("MC_ROUTE_CODER_LITE", ""),
|
||||
"heavy_chars": int(os.environ.get("MC_GATEWAY_HEAVY_CHARS", "8000")),
|
||||
"coding_escalate_chars": int(os.environ.get("MC_CODING_ESCALATE_CHARS", "120000")),
|
||||
"fast_no_think": _env_bool("MC_FAST_NO_THINK", "1"),
|
||||
}
|
||||
|
||||
# Feld-Spezifikation für die UI (Typ + Grenzen + Label). Treibt Editor & Validierung.
|
||||
FIELDS: list[dict] = [
|
||||
{"key": "fast", "label": "fast-Alias (chat: Standard)", "type": "str"},
|
||||
{"key": "heavy", "label": "heavy-Alias (chat: lang/komplex)", "type": "str"},
|
||||
{"key": "coder", "label": "coder-Alias (coding: stark / Eskalation)", "type": "str"},
|
||||
{"key": "coder_lite", "label": "coder-lite-Alias (coding: schneller Default; leer = aus)", "type": "str"},
|
||||
{"key": "heavy_chars", "label": "chat → heavy ab N Zeichen", "type": "int", "min": 500, "max": 1_000_000},
|
||||
{"key": "coding_escalate_chars", "label": "coding → starker Coder ab N Zeichen", "type": "int", "min": 1000, "max": 4_000_000},
|
||||
{"key": "fast_no_think", "label": "fast-Spur: Thinking aus (flotte Antworten)", "type": "bool"},
|
||||
]
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_CACHE: dict = {"mtime": None, "policy": None}
|
||||
|
||||
|
||||
def _read_file() -> dict:
|
||||
try:
|
||||
with open(POLICY_PATH, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _coerce(patch: dict) -> dict:
|
||||
"""Nur bekannte Keys, typ-/bereichsvalidiert. Wirft ValueError bei ungültigen Werten."""
|
||||
spec = {f["key"]: f for f in FIELDS}
|
||||
out: dict = {}
|
||||
for k, v in (patch or {}).items():
|
||||
f = spec.get(k)
|
||||
if not f:
|
||||
continue # unbekannte Keys still verwerfen
|
||||
if f["type"] == "int":
|
||||
iv = int(v)
|
||||
lo, hi = f.get("min", 1), f.get("max", 10**9)
|
||||
if not (lo <= iv <= hi):
|
||||
raise ValueError(f"{k}={iv} außerhalb [{lo}, {hi}]")
|
||||
out[k] = iv
|
||||
elif f["type"] == "bool":
|
||||
out[k] = bool(v)
|
||||
else: # str
|
||||
sv = str(v).strip()
|
||||
if k != "coder_lite" and not sv:
|
||||
raise ValueError(f"{k} darf nicht leer sein")
|
||||
out[k] = sv
|
||||
return out
|
||||
|
||||
|
||||
def _coerce_safe(patch: dict) -> dict:
|
||||
"""Wie _coerce, aber schluckt Fehler — kaputte Datei darf den Betrieb nicht stoppen."""
|
||||
try:
|
||||
return _coerce(patch)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def load_policy() -> dict:
|
||||
"""Aktuelle Policy (Datei über DEFAULTS gemerged). Hot-reload via mtime-Cache, pro Request billig."""
|
||||
try:
|
||||
mtime = POLICY_PATH.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
with _LOCK:
|
||||
if _CACHE["policy"] is None or _CACHE["mtime"] != mtime:
|
||||
merged = {**DEFAULTS}
|
||||
if mtime is not None:
|
||||
merged.update(_coerce_safe(_read_file()))
|
||||
_CACHE["mtime"] = mtime
|
||||
_CACHE["policy"] = merged
|
||||
return dict(_CACHE["policy"])
|
||||
|
||||
|
||||
def save_policy(patch: dict) -> dict:
|
||||
"""Validiert + persistiert atomar. Gibt die neue, vollständige Policy zurück."""
|
||||
clean = _coerce(patch) # wirft bei ungültigem Input
|
||||
with _LOCK:
|
||||
current = {**DEFAULTS, **_coerce_safe(_read_file()), **clean}
|
||||
POLICY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = POLICY_PATH.with_suffix(".json.tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(current, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp, POLICY_PATH)
|
||||
_CACHE["mtime"] = None # nächster load_policy() lädt frisch
|
||||
_CACHE["policy"] = None
|
||||
return current
|
||||
|
||||
|
||||
def policy_meta() -> dict:
|
||||
"""Für den UI-Editor: aktuelle Werte + Defaults (für „Zurücksetzen“) + Feld-Spezifikation."""
|
||||
return {"policy": load_policy(), "defaults": dict(DEFAULTS), "fields": FIELDS}
|
||||
"""
|
||||
UI-editierbare Routing-Policy für die Gateway-Lanes (coding/chat).
|
||||
|
||||
Persistiert als JSON unter MC_ROUTING_POLICY_PATH (Default MODELS_DIR/mc2-routing.json —
|
||||
gleiche Konvention wie mc2-discover.json). **Hot-reload:** load_policy() liest die Datei nur
|
||||
bei Änderung neu (mtime-Cache) → UI-Edits greifen ohne Dienst-Neustart. Die Env-Vars (bisher
|
||||
einzige Stellschraube in router_logic.py) bleiben als Defaults/Fallback erhalten.
|
||||
|
||||
Bewusst NICHT editierbar (v1): die Regex-Keyword-Listen (heavy/coding-heavy/code-hint) — die
|
||||
bleiben in router_logic.py im Code.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from config import MODELS_DIR
|
||||
|
||||
POLICY_PATH = Path(os.environ.get("MC_ROUTING_POLICY_PATH", str(MODELS_DIR / "mc2-routing.json")))
|
||||
|
||||
|
||||
def _env_bool(name: str, default: str) -> bool:
|
||||
return os.environ.get(name, default) not in ("0", "false", "")
|
||||
|
||||
|
||||
# Defaults aus den Env-Vars — Quelle der Wahrheit, solange keine Policy-Datei existiert.
|
||||
DEFAULTS: dict = {
|
||||
"fast": os.environ.get("MC_ROUTE_FAST", "fast"),
|
||||
"heavy": os.environ.get("MC_ROUTE_HEAVY", "heavy"),
|
||||
"coder": os.environ.get("MC_ROUTE_CODER", "coder"),
|
||||
"coder_lite": os.environ.get("MC_ROUTE_CODER_LITE", ""),
|
||||
"heavy_chars": int(os.environ.get("MC_GATEWAY_HEAVY_CHARS", "8000")),
|
||||
"coding_escalate_chars": int(os.environ.get("MC_CODING_ESCALATE_CHARS", "120000")),
|
||||
"fast_no_think": _env_bool("MC_FAST_NO_THINK", "1"),
|
||||
}
|
||||
|
||||
# Feld-Spezifikation für die UI (Typ + Grenzen + Label). Treibt Editor & Validierung.
|
||||
FIELDS: list[dict] = [
|
||||
{"key": "fast", "label": "fast-Alias (chat: Standard)", "type": "str"},
|
||||
{"key": "heavy", "label": "heavy-Alias (chat: lang/komplex)", "type": "str"},
|
||||
{"key": "coder", "label": "coder-Alias (coding: stark / Eskalation)", "type": "str"},
|
||||
{"key": "coder_lite", "label": "coder-lite-Alias (coding: schneller Default; leer = aus)", "type": "str"},
|
||||
{"key": "heavy_chars", "label": "chat → heavy ab N Zeichen", "type": "int", "min": 500, "max": 1_000_000},
|
||||
{"key": "coding_escalate_chars", "label": "coding → starker Coder ab N Zeichen", "type": "int", "min": 1000, "max": 4_000_000},
|
||||
{"key": "fast_no_think", "label": "fast-Spur: Thinking aus (flotte Antworten)", "type": "bool"},
|
||||
]
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_CACHE: dict = {"mtime": None, "policy": None}
|
||||
|
||||
|
||||
def _read_file() -> dict:
|
||||
try:
|
||||
with open(POLICY_PATH, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _coerce(patch: dict) -> dict:
|
||||
"""Nur bekannte Keys, typ-/bereichsvalidiert. Wirft ValueError bei ungültigen Werten."""
|
||||
spec = {f["key"]: f for f in FIELDS}
|
||||
out: dict = {}
|
||||
for k, v in (patch or {}).items():
|
||||
f = spec.get(k)
|
||||
if not f:
|
||||
continue # unbekannte Keys still verwerfen
|
||||
if f["type"] == "int":
|
||||
iv = int(v)
|
||||
lo, hi = f.get("min", 1), f.get("max", 10**9)
|
||||
if not (lo <= iv <= hi):
|
||||
raise ValueError(f"{k}={iv} außerhalb [{lo}, {hi}]")
|
||||
out[k] = iv
|
||||
elif f["type"] == "bool":
|
||||
out[k] = bool(v)
|
||||
else: # str
|
||||
sv = str(v).strip()
|
||||
if k != "coder_lite" and not sv:
|
||||
raise ValueError(f"{k} darf nicht leer sein")
|
||||
out[k] = sv
|
||||
return out
|
||||
|
||||
|
||||
def _coerce_safe(patch: dict) -> dict:
|
||||
"""Wie _coerce, aber schluckt Fehler — kaputte Datei darf den Betrieb nicht stoppen."""
|
||||
try:
|
||||
return _coerce(patch)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def load_policy() -> dict:
|
||||
"""Aktuelle Policy (Datei über DEFAULTS gemerged). Hot-reload via mtime-Cache, pro Request billig."""
|
||||
try:
|
||||
mtime = POLICY_PATH.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
with _LOCK:
|
||||
if _CACHE["policy"] is None or _CACHE["mtime"] != mtime:
|
||||
merged = {**DEFAULTS}
|
||||
if mtime is not None:
|
||||
merged.update(_coerce_safe(_read_file()))
|
||||
_CACHE["mtime"] = mtime
|
||||
_CACHE["policy"] = merged
|
||||
return dict(_CACHE["policy"])
|
||||
|
||||
|
||||
def save_policy(patch: dict) -> dict:
|
||||
"""Validiert + persistiert atomar. Gibt die neue, vollständige Policy zurück."""
|
||||
clean = _coerce(patch) # wirft bei ungültigem Input
|
||||
with _LOCK:
|
||||
current = {**DEFAULTS, **_coerce_safe(_read_file()), **clean}
|
||||
POLICY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = POLICY_PATH.with_suffix(".json.tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(current, f, ensure_ascii=False, indent=2)
|
||||
os.replace(tmp, POLICY_PATH)
|
||||
_CACHE["mtime"] = None # nächster load_policy() lädt frisch
|
||||
_CACHE["policy"] = None
|
||||
return current
|
||||
|
||||
|
||||
def policy_meta() -> dict:
|
||||
"""Für den UI-Editor: aktuelle Werte + Defaults (für „Zurücksetzen“) + Feld-Spezifikation."""
|
||||
return {"policy": load_policy(), "defaults": dict(DEFAULTS), "fields": FIELDS}
|
||||
|
||||
@@ -1,99 +1,99 @@
|
||||
"""Token-Statistik (Verbrauch je Modell) mit gedrosseltem Persistieren.
|
||||
|
||||
Früher wurde bei JEDEM Request die komplette JSON-Datei gelesen und geschrieben
|
||||
(Disk-Thrash). Jetzt: einmaliges Laden in einen In-Memory-Cache, Inkremente laufen
|
||||
gegen den Cache, Persistieren passiert höchstens alle FLUSH_INTERVAL Sekunden sowie
|
||||
beim Prozess-Ende (atexit). Lesen liefert immer den aktuellen (auch ungeflushten) Stand.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from config import HERMES_HOME
|
||||
|
||||
STATS_FILE = HERMES_HOME / "token_stats.json"
|
||||
FLUSH_INTERVAL = 5.0 # Sekunden zwischen Disk-Writes
|
||||
# Baseline (repräsentiert Verbrauch vor dem modellspezifischen Logging).
|
||||
_BASELINE = {"prompt_tokens": 718400, "completion_tokens": 324200, "models": {}}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_lock = threading.Lock()
|
||||
_stats: dict | None = None
|
||||
_dirty = False
|
||||
_last_flush = 0.0
|
||||
|
||||
|
||||
def _load_from_disk() -> dict:
|
||||
if not STATS_FILE.exists():
|
||||
return dict(_BASELINE)
|
||||
try:
|
||||
with open(STATS_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data.setdefault("prompt_tokens", 0)
|
||||
data.setdefault("completion_tokens", 0)
|
||||
data.setdefault("models", {})
|
||||
return data
|
||||
except (OSError, json.JSONDecodeError):
|
||||
log.warning("token_stats: Laden fehlgeschlagen, nutze Baseline", exc_info=True)
|
||||
return dict(_BASELINE)
|
||||
|
||||
|
||||
def _ensure_loaded() -> dict:
|
||||
global _stats
|
||||
if _stats is None:
|
||||
_stats = _load_from_disk()
|
||||
return _stats
|
||||
|
||||
|
||||
def _write(stats: dict) -> None:
|
||||
try:
|
||||
STATS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = STATS_FILE.with_suffix(".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(stats, f)
|
||||
tmp.replace(STATS_FILE)
|
||||
except OSError:
|
||||
log.warning("token_stats: Schreiben fehlgeschlagen", exc_info=True)
|
||||
|
||||
|
||||
def get_stats() -> dict:
|
||||
"""Aktueller Stand (inkl. noch nicht geflushter Inkremente) als Kopie."""
|
||||
with _lock:
|
||||
return json.loads(json.dumps(_ensure_loaded()))
|
||||
|
||||
|
||||
def increment_tokens(prompt: int, completion: int, model: str | None = None) -> None:
|
||||
"""Tokens im Cache verbuchen; gedrosselt auf Disk persistieren."""
|
||||
global _dirty, _last_flush
|
||||
with _lock:
|
||||
stats = _ensure_loaded()
|
||||
stats["prompt_tokens"] += prompt
|
||||
stats["completion_tokens"] += completion
|
||||
if model:
|
||||
m = stats.setdefault("models", {}).setdefault(
|
||||
model.lower(), {"prompt": 0, "completion": 0})
|
||||
m["prompt"] += prompt
|
||||
m["completion"] += completion
|
||||
_dirty = True
|
||||
now = time.monotonic()
|
||||
if now - _last_flush >= FLUSH_INTERVAL:
|
||||
_write(stats)
|
||||
_dirty = False
|
||||
_last_flush = now
|
||||
|
||||
|
||||
def flush() -> None:
|
||||
"""Ungeschriebene Inkremente sofort persistieren (z.B. beim Shutdown)."""
|
||||
global _dirty
|
||||
with _lock:
|
||||
if _dirty and _stats is not None:
|
||||
_write(_stats)
|
||||
_dirty = False
|
||||
|
||||
|
||||
atexit.register(flush)
|
||||
"""Token-Statistik (Verbrauch je Modell) mit gedrosseltem Persistieren.
|
||||
|
||||
Früher wurde bei JEDEM Request die komplette JSON-Datei gelesen und geschrieben
|
||||
(Disk-Thrash). Jetzt: einmaliges Laden in einen In-Memory-Cache, Inkremente laufen
|
||||
gegen den Cache, Persistieren passiert höchstens alle FLUSH_INTERVAL Sekunden sowie
|
||||
beim Prozess-Ende (atexit). Lesen liefert immer den aktuellen (auch ungeflushten) Stand.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from config import HERMES_HOME
|
||||
|
||||
STATS_FILE = HERMES_HOME / "token_stats.json"
|
||||
FLUSH_INTERVAL = 5.0 # Sekunden zwischen Disk-Writes
|
||||
# Baseline (repräsentiert Verbrauch vor dem modellspezifischen Logging).
|
||||
_BASELINE = {"prompt_tokens": 718400, "completion_tokens": 324200, "models": {}}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_lock = threading.Lock()
|
||||
_stats: dict | None = None
|
||||
_dirty = False
|
||||
_last_flush = 0.0
|
||||
|
||||
|
||||
def _load_from_disk() -> dict:
|
||||
if not STATS_FILE.exists():
|
||||
return dict(_BASELINE)
|
||||
try:
|
||||
with open(STATS_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data.setdefault("prompt_tokens", 0)
|
||||
data.setdefault("completion_tokens", 0)
|
||||
data.setdefault("models", {})
|
||||
return data
|
||||
except (OSError, json.JSONDecodeError):
|
||||
log.warning("token_stats: Laden fehlgeschlagen, nutze Baseline", exc_info=True)
|
||||
return dict(_BASELINE)
|
||||
|
||||
|
||||
def _ensure_loaded() -> dict:
|
||||
global _stats
|
||||
if _stats is None:
|
||||
_stats = _load_from_disk()
|
||||
return _stats
|
||||
|
||||
|
||||
def _write(stats: dict) -> None:
|
||||
try:
|
||||
STATS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = STATS_FILE.with_suffix(".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(stats, f)
|
||||
tmp.replace(STATS_FILE)
|
||||
except OSError:
|
||||
log.warning("token_stats: Schreiben fehlgeschlagen", exc_info=True)
|
||||
|
||||
|
||||
def get_stats() -> dict:
|
||||
"""Aktueller Stand (inkl. noch nicht geflushter Inkremente) als Kopie."""
|
||||
with _lock:
|
||||
return json.loads(json.dumps(_ensure_loaded()))
|
||||
|
||||
|
||||
def increment_tokens(prompt: int, completion: int, model: str | None = None) -> None:
|
||||
"""Tokens im Cache verbuchen; gedrosselt auf Disk persistieren."""
|
||||
global _dirty, _last_flush
|
||||
with _lock:
|
||||
stats = _ensure_loaded()
|
||||
stats["prompt_tokens"] += prompt
|
||||
stats["completion_tokens"] += completion
|
||||
if model:
|
||||
m = stats.setdefault("models", {}).setdefault(
|
||||
model.lower(), {"prompt": 0, "completion": 0})
|
||||
m["prompt"] += prompt
|
||||
m["completion"] += completion
|
||||
_dirty = True
|
||||
now = time.monotonic()
|
||||
if now - _last_flush >= FLUSH_INTERVAL:
|
||||
_write(stats)
|
||||
_dirty = False
|
||||
_last_flush = now
|
||||
|
||||
|
||||
def flush() -> None:
|
||||
"""Ungeschriebene Inkremente sofort persistieren (z.B. beim Shutdown)."""
|
||||
global _dirty
|
||||
with _lock:
|
||||
if _dirty and _stats is not None:
|
||||
_write(_stats)
|
||||
_dirty = False
|
||||
|
||||
|
||||
atexit.register(flush)
|
||||
|
||||
+129
-129
@@ -1,129 +1,129 @@
|
||||
"""
|
||||
Hält das ganze WARM-SET (Hirn + Augen/vision + Gedächtnis/embed) dauerhaft warm.
|
||||
|
||||
Hintergrund: llama-swap ist EIN-Gruppen-resident — lädt ein on-demand-Modell außerhalb
|
||||
der `brains`-Gruppe, wird die ganze Gruppe verdrängt. `persist: true` verhindert nur
|
||||
Idle-Unload, NICHT die Gruppen-Verdrängung; auch ein `-watch-config`-Reload (jede Config-
|
||||
Änderung/Deploy) verwirft das Set, ohne llama-swaps ExecStartPost-Warmup neu auszulösen.
|
||||
Dieser Wächter schließt die Lücke: ist die Box idle (nichts geladen), lädt er das ganze
|
||||
Set über deploy/warmup.sh nach — NICHT nur das Hirn (sonst blieben Augen+embed kalt, live
|
||||
vom Review-Wächter beobachtet). Während aktiver Last (irgendetwas geladen) hält er sich
|
||||
raus, verdrängt also nie ein gerade genutztes Modell.
|
||||
|
||||
warmup.sh deckt korrekt ab: fast+vision über /v1/chat/completions, embed über /v1/embeddings
|
||||
(anderer Endpunkt!) und das Vorkauen des ~70-KB-Agent-Prompts. Es ist selbst-detachend.
|
||||
|
||||
Abschaltbar/justierbar via Env: MC_REWARM_ENABLED=0, MC_REWARM_INTERVAL. Welche Modelle
|
||||
warmup.sh lädt: MC_WARMUP_MODELS (Default 'fast vision') + MC_WARMUP_EMBED (Default 'embed').
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import httpx
|
||||
|
||||
from config import LLAMA_SWAP_URL
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ENABLED = os.environ.get("MC_REWARM_ENABLED", "1") != "0"
|
||||
INTERVAL = int(os.environ.get("MC_REWARM_INTERVAL", "90")) # Sekunden zwischen Checks
|
||||
START_DELAY = int(os.environ.get("MC_REWARM_START_DELAY", "25"))
|
||||
NUDGE_GRACE = int(os.environ.get("MC_REWARM_NUDGE_GRACE", "3")) # llama-swap den Reload abschließen lassen
|
||||
|
||||
# Das geteilte Warm-Skript (auch llama-swaps ExecStartPost) — EINE Quelle für „was ist das
|
||||
# Warm-Set und wie wärmt man es korrekt", statt hier eine zweite, ärmere Logik zu pflegen.
|
||||
_WARMUP_SH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "deploy", "warmup.sh"))
|
||||
|
||||
# Wecksignal für einen sofortigen Vorwärm-Check (statt bis zum nächsten INTERVAL-Tick zu warten).
|
||||
# Wird von write_config() nach einer Config-Änderung gesetzt: llama-swap (-watch-config) lädt die
|
||||
# neue Config und verwirft dabei ALLE Modelle inkl. Hirn — ohne Nudge bliebe es bis zu INTERVAL
|
||||
# Sekunden kalt liegen, bis der nächste Tick oder eine Anfrage es wieder lädt.
|
||||
_loop: asyncio.AbstractEventLoop | None = None
|
||||
_wake: asyncio.Event | None = None
|
||||
|
||||
|
||||
def nudge() -> None:
|
||||
"""Threadsicher: bittet den Wächter, nach einem Config-Reload bald vorzuwärmen. No-op,
|
||||
solange der Wächter (noch) nicht läuft."""
|
||||
if _loop is not None and _wake is not None and not _loop.is_closed():
|
||||
try:
|
||||
_loop.call_soon_threadsafe(_wake.set)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def _run_warmup() -> bool:
|
||||
"""Volles Warm-Set via deploy/warmup.sh nachladen (fast+vision über /v1/chat/completions,
|
||||
embed über /v1/embeddings, plus Agent-Prompt-Prefill). Selbst-detachend, blockiert nicht.
|
||||
False, wenn das Skript fehlt."""
|
||||
if not os.path.exists(_WARMUP_SH):
|
||||
log.warning("rewarm: warmup.sh nicht gefunden (%s) — kein Nachwärmen möglich", _WARMUP_SH)
|
||||
return False
|
||||
try:
|
||||
subprocess.Popen(["bash", _WARMUP_SH],
|
||||
env={**os.environ, "MC_LLAMA_SWAP_URL": LLAMA_SWAP_URL},
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
return True
|
||||
except Exception:
|
||||
log.debug("rewarm: warmup.sh-Start fehlgeschlagen", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _warmset_members() -> set[str]:
|
||||
"""Soll-Warm-Set = Mitglieder aller ko-residenten Gruppen (swap:false bzw. persist/persistent)."""
|
||||
try:
|
||||
from services import llamaswap
|
||||
groups = llamaswap.list_groups() or {}
|
||||
except Exception:
|
||||
return set()
|
||||
want: set[str] = set()
|
||||
for g in groups.values():
|
||||
if isinstance(g, dict) and (g.get("swap") is False or g.get("persist") or g.get("persistent")):
|
||||
want.update(g.get("members") or [])
|
||||
return want
|
||||
|
||||
|
||||
async def _warmset_missing() -> list[str] | None:
|
||||
"""Warm-Set-Mitglieder, die NICHT 'ready' in /running sind. [] = alles warm, None = /running
|
||||
nicht lesbar. Erkennt auch TEIL-Kälte (nur Hirn warm, Augen/embed rausgefallen) — genau der
|
||||
Fall, der nach einem watch-config-Reload/Deploy auftritt."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as c:
|
||||
r = await c.get(f"{LLAMA_SWAP_URL}/running")
|
||||
data = r.json() or {}
|
||||
except Exception:
|
||||
return None
|
||||
ready = {str(x.get("model")) for x in (data.get("running") or []) if x.get("state") == "ready"}
|
||||
want = _warmset_members()
|
||||
if not want: # Set unbekannt → alte Heuristik: nur bei ganz leer
|
||||
return [] if ready else ["<leer>"]
|
||||
return [m for m in want if m not in ready]
|
||||
|
||||
|
||||
async def rewarm_loop() -> None:
|
||||
"""Endlos-Schleife (Hintergrund-Task): hält das ganze Warm-Set warm. Prüft periodisch, ob ein
|
||||
Mitglied fehlt (Teil-Kälte!), und sofort nach einem Config-Reload-Nudge — lädt via warmup.sh nach."""
|
||||
global _loop, _wake
|
||||
_loop = asyncio.get_running_loop()
|
||||
_wake = asyncio.Event()
|
||||
await asyncio.sleep(START_DELAY) # Box/Engine nach MC-Start setzen lassen
|
||||
while True:
|
||||
try:
|
||||
missing = await _warmset_missing()
|
||||
if missing:
|
||||
log.info("rewarm: Warm-Set unvollständig (%s) → warmup.sh", ", ".join(missing))
|
||||
_run_warmup()
|
||||
except Exception:
|
||||
log.debug("rewarm: Tick fehlgeschlagen", exc_info=True)
|
||||
# Bis zum nächsten Tick warten ODER sofort auf einen Config-Reload-Nudge reagieren.
|
||||
try:
|
||||
await asyncio.wait_for(_wake.wait(), timeout=INTERVAL)
|
||||
_wake.clear()
|
||||
await asyncio.sleep(NUDGE_GRACE) # llama-swap den Reload abschließen lassen
|
||||
log.info("rewarm: Config-Reload → warmup.sh (volles Warm-Set nachladen)")
|
||||
_run_warmup()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
"""
|
||||
Hält das ganze WARM-SET (Hirn + Augen/vision + Gedächtnis/embed) dauerhaft warm.
|
||||
|
||||
Hintergrund: llama-swap ist EIN-Gruppen-resident — lädt ein on-demand-Modell außerhalb
|
||||
der `brains`-Gruppe, wird die ganze Gruppe verdrängt. `persist: true` verhindert nur
|
||||
Idle-Unload, NICHT die Gruppen-Verdrängung; auch ein `-watch-config`-Reload (jede Config-
|
||||
Änderung/Deploy) verwirft das Set, ohne llama-swaps ExecStartPost-Warmup neu auszulösen.
|
||||
Dieser Wächter schließt die Lücke: ist die Box idle (nichts geladen), lädt er das ganze
|
||||
Set über deploy/warmup.sh nach — NICHT nur das Hirn (sonst blieben Augen+embed kalt, live
|
||||
vom Review-Wächter beobachtet). Während aktiver Last (irgendetwas geladen) hält er sich
|
||||
raus, verdrängt also nie ein gerade genutztes Modell.
|
||||
|
||||
warmup.sh deckt korrekt ab: fast+vision über /v1/chat/completions, embed über /v1/embeddings
|
||||
(anderer Endpunkt!) und das Vorkauen des ~70-KB-Agent-Prompts. Es ist selbst-detachend.
|
||||
|
||||
Abschaltbar/justierbar via Env: MC_REWARM_ENABLED=0, MC_REWARM_INTERVAL. Welche Modelle
|
||||
warmup.sh lädt: MC_WARMUP_MODELS (Default 'fast vision') + MC_WARMUP_EMBED (Default 'embed').
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import httpx
|
||||
|
||||
from config import LLAMA_SWAP_URL
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ENABLED = os.environ.get("MC_REWARM_ENABLED", "1") != "0"
|
||||
INTERVAL = int(os.environ.get("MC_REWARM_INTERVAL", "90")) # Sekunden zwischen Checks
|
||||
START_DELAY = int(os.environ.get("MC_REWARM_START_DELAY", "25"))
|
||||
NUDGE_GRACE = int(os.environ.get("MC_REWARM_NUDGE_GRACE", "3")) # llama-swap den Reload abschließen lassen
|
||||
|
||||
# Das geteilte Warm-Skript (auch llama-swaps ExecStartPost) — EINE Quelle für „was ist das
|
||||
# Warm-Set und wie wärmt man es korrekt", statt hier eine zweite, ärmere Logik zu pflegen.
|
||||
_WARMUP_SH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "deploy", "warmup.sh"))
|
||||
|
||||
# Wecksignal für einen sofortigen Vorwärm-Check (statt bis zum nächsten INTERVAL-Tick zu warten).
|
||||
# Wird von write_config() nach einer Config-Änderung gesetzt: llama-swap (-watch-config) lädt die
|
||||
# neue Config und verwirft dabei ALLE Modelle inkl. Hirn — ohne Nudge bliebe es bis zu INTERVAL
|
||||
# Sekunden kalt liegen, bis der nächste Tick oder eine Anfrage es wieder lädt.
|
||||
_loop: asyncio.AbstractEventLoop | None = None
|
||||
_wake: asyncio.Event | None = None
|
||||
|
||||
|
||||
def nudge() -> None:
|
||||
"""Threadsicher: bittet den Wächter, nach einem Config-Reload bald vorzuwärmen. No-op,
|
||||
solange der Wächter (noch) nicht läuft."""
|
||||
if _loop is not None and _wake is not None and not _loop.is_closed():
|
||||
try:
|
||||
_loop.call_soon_threadsafe(_wake.set)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def _run_warmup() -> bool:
|
||||
"""Volles Warm-Set via deploy/warmup.sh nachladen (fast+vision über /v1/chat/completions,
|
||||
embed über /v1/embeddings, plus Agent-Prompt-Prefill). Selbst-detachend, blockiert nicht.
|
||||
False, wenn das Skript fehlt."""
|
||||
if not os.path.exists(_WARMUP_SH):
|
||||
log.warning("rewarm: warmup.sh nicht gefunden (%s) — kein Nachwärmen möglich", _WARMUP_SH)
|
||||
return False
|
||||
try:
|
||||
subprocess.Popen(["bash", _WARMUP_SH],
|
||||
env={**os.environ, "MC_LLAMA_SWAP_URL": LLAMA_SWAP_URL},
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
return True
|
||||
except Exception:
|
||||
log.debug("rewarm: warmup.sh-Start fehlgeschlagen", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _warmset_members() -> set[str]:
|
||||
"""Soll-Warm-Set = Mitglieder aller ko-residenten Gruppen (swap:false bzw. persist/persistent)."""
|
||||
try:
|
||||
from services import llamaswap
|
||||
groups = llamaswap.list_groups() or {}
|
||||
except Exception:
|
||||
return set()
|
||||
want: set[str] = set()
|
||||
for g in groups.values():
|
||||
if isinstance(g, dict) and (g.get("swap") is False or g.get("persist") or g.get("persistent")):
|
||||
want.update(g.get("members") or [])
|
||||
return want
|
||||
|
||||
|
||||
async def _warmset_missing() -> list[str] | None:
|
||||
"""Warm-Set-Mitglieder, die NICHT 'ready' in /running sind. [] = alles warm, None = /running
|
||||
nicht lesbar. Erkennt auch TEIL-Kälte (nur Hirn warm, Augen/embed rausgefallen) — genau der
|
||||
Fall, der nach einem watch-config-Reload/Deploy auftritt."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as c:
|
||||
r = await c.get(f"{LLAMA_SWAP_URL}/running")
|
||||
data = r.json() or {}
|
||||
except Exception:
|
||||
return None
|
||||
ready = {str(x.get("model")) for x in (data.get("running") or []) if x.get("state") == "ready"}
|
||||
want = _warmset_members()
|
||||
if not want: # Set unbekannt → alte Heuristik: nur bei ganz leer
|
||||
return [] if ready else ["<leer>"]
|
||||
return [m for m in want if m not in ready]
|
||||
|
||||
|
||||
async def rewarm_loop() -> None:
|
||||
"""Endlos-Schleife (Hintergrund-Task): hält das ganze Warm-Set warm. Prüft periodisch, ob ein
|
||||
Mitglied fehlt (Teil-Kälte!), und sofort nach einem Config-Reload-Nudge — lädt via warmup.sh nach."""
|
||||
global _loop, _wake
|
||||
_loop = asyncio.get_running_loop()
|
||||
_wake = asyncio.Event()
|
||||
await asyncio.sleep(START_DELAY) # Box/Engine nach MC-Start setzen lassen
|
||||
while True:
|
||||
try:
|
||||
missing = await _warmset_missing()
|
||||
if missing:
|
||||
log.info("rewarm: Warm-Set unvollständig (%s) → warmup.sh", ", ".join(missing))
|
||||
_run_warmup()
|
||||
except Exception:
|
||||
log.debug("rewarm: Tick fehlgeschlagen", exc_info=True)
|
||||
# Bis zum nächsten Tick warten ODER sofort auf einen Config-Reload-Nudge reagieren.
|
||||
try:
|
||||
await asyncio.wait_for(_wake.wait(), timeout=INTERVAL)
|
||||
_wake.clear()
|
||||
await asyncio.sleep(NUDGE_GRACE) # llama-swap den Reload abschließen lassen
|
||||
log.info("rewarm: Config-Reload → warmup.sh (volles Warm-Set nachladen)")
|
||||
_run_warmup()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user