Hitonabi
b75335dafd
- news.py: Git-Release-Feeds raus (b9745-Rauschen), Qualitaetsquellen rein (HuggingFace, r/LocalLLaMA, Simon Willison, Latent Space, Ahead of AI, The Batch). - cookbook.py: /upgrades (matcht installierte alte Modelle gegen UPGRADES-Map) + /install-model (einzelnes Modell tauschen, GGUF dynamisch, optimaler ctx). - news.js: 'Fuer dich: Upgrades'-Sektion oben (Modell X ersetzt dein Y, weil ... [Installieren]). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
238 lines
9.0 KiB
Python
238 lines
9.0 KiB
Python
"""
|
|
Cookbook Router: Verbindet die HuggingFace API mit der Odysseus-Hardware-Berechnung.
|
|
"""
|
|
|
|
import httpx
|
|
import re
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|
from pydantic import BaseModel
|
|
import psutil
|
|
|
|
from ruamel.yaml.scalarstring import LiteralScalarString
|
|
|
|
from auth import auth
|
|
from hw_math import evaluate_fit, max_ctx_for
|
|
from config import MODELS_DIR, CMD_TEMPLATE, DEFAULT_TTL
|
|
from llamaswap import read_config, write_config
|
|
from jobengine import start_job, JOBS
|
|
from recipes import RECIPES, UPGRADES
|
|
|
|
router = APIRouter(prefix="/api/cookbook", dependencies=[Depends(auth)])
|
|
|
|
_FIT_ORDER = {"perfect": 0, "marginal": 1, "too_tight": 2}
|
|
|
|
class AnalyzeRequest(BaseModel):
|
|
repo_id: str
|
|
ctx: int = 8192
|
|
|
|
class EvaluateRequest(BaseModel):
|
|
params_b: float
|
|
quant: str
|
|
ctx: int
|
|
|
|
class InstallRecipeReq(BaseModel):
|
|
recipe_id: str
|
|
hf_token: str | None = None
|
|
|
|
class InstallModelReq(BaseModel):
|
|
repo: str
|
|
role: str
|
|
params_b: float
|
|
quant: str = "Q4_K_M"
|
|
hf_token: str | None = None
|
|
|
|
def extract_params_b(repo_id: str) -> float:
|
|
"""Extrahiert die Parametergröße (in Milliarden) aus dem Repo-Namen."""
|
|
# z.B. Qwen2.5-Coder-32B -> 32
|
|
# 8x7B -> 56 (MoE)
|
|
moe = re.search(r"(\d+)x(\d+(?:\.\d+)?)[bB]", repo_id)
|
|
if moe:
|
|
return float(moe.group(1)) * float(moe.group(2))
|
|
m = re.search(r"(\d+(?:\.\d+)?)[bB](?![a-zA-Z])", repo_id)
|
|
if m:
|
|
return float(m.group(1))
|
|
return 7.0 # Fallback
|
|
|
|
def extract_quant(filename: str) -> str:
|
|
m = re.search(r"(Q\d_[A-Z0-9_]+|IQ\d_[A-Z0-9_]+|FP16|BF16)", filename, re.IGNORECASE)
|
|
return m.group(1).upper() if m else "Q4_K_M"
|
|
|
|
@router.post("/analyze")
|
|
async def analyze_repo(req: AnalyzeRequest):
|
|
"""Holt die GGUF Dateien von HuggingFace und berechnet den Hardware-Fit."""
|
|
url = f"https://huggingface.co/api/models/{req.repo_id}/tree/main"
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
try:
|
|
resp = await client.get(url, timeout=10.0)
|
|
resp.raise_for_status()
|
|
tree = resp.json()
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"HuggingFace Fehler: {str(e)}")
|
|
|
|
gguf_files = [f["path"] for f in tree if f.get("path", "").endswith(".gguf")]
|
|
|
|
if not gguf_files:
|
|
return {"files": []}
|
|
|
|
params_b = extract_params_b(req.repo_id)
|
|
|
|
# Ermittle RAM des Systems (da APU = Shared Memory)
|
|
ram_gb = psutil.virtual_memory().total / (1024**3)
|
|
|
|
results = []
|
|
for f in gguf_files:
|
|
quant = extract_quant(f)
|
|
fit = evaluate_fit(params_b, quant, req.ctx, ram_gb)
|
|
|
|
# Priority-Score, um den besten Fit an oberste Stelle zu setzen.
|
|
# "Q4_K_M" ist oft der Sweetspot.
|
|
priority = 0
|
|
if fit["level"] == "perfect":
|
|
priority += 10
|
|
if quant == "Q4_K_M": priority += 5
|
|
elif quant.startswith("Q4"): priority += 4
|
|
elif quant.startswith("Q5"): priority += 3
|
|
|
|
results.append({
|
|
"filename": f,
|
|
"quant": quant,
|
|
"fit": fit,
|
|
"optimal_ctx": max_ctx_for(params_b, quant, ram_gb),
|
|
"priority": priority
|
|
})
|
|
|
|
# Sortieren: Highest priority first, dann nach tps (schnellste zuerst)
|
|
results.sort(key=lambda x: (x["priority"], x["fit"]["tps"]), reverse=True)
|
|
|
|
return {
|
|
"repo": req.repo_id,
|
|
"params_b": params_b,
|
|
"sys_ram_gb": round(ram_gb, 1),
|
|
"files": results
|
|
}
|
|
|
|
@router.post("/evaluate")
|
|
def evaluate_single(req: EvaluateRequest):
|
|
ram_gb = psutil.virtual_memory().total / (1024**3)
|
|
fit = evaluate_fit(req.params_b, req.quant, req.ctx, ram_gb)
|
|
fit["optimal_ctx"] = max_ctx_for(req.params_b, req.quant, ram_gb)
|
|
return fit
|
|
|
|
|
|
@router.get("/recipes")
|
|
def recipes():
|
|
"""Use-Case-Setups mit Hardware-Fit pro Modell + Stack-Gesamturteil. Da llama-swap nur EIN
|
|
Modell gleichzeitig lädt, ist das Stack-Urteil der schlechteste (= größte) Einzel-Fit."""
|
|
ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
out = []
|
|
for r in RECIPES:
|
|
models, worst = [], "perfect"
|
|
for m in r["models"]:
|
|
fit = evaluate_fit(m["params_b"], m["quant"], 8192, ram_gb)
|
|
models.append({**m, "fit": fit, "optimal_ctx": max_ctx_for(m["params_b"], m["quant"], ram_gb)})
|
|
if _FIT_ORDER[fit["level"]] > _FIT_ORDER[worst]:
|
|
worst = fit["level"]
|
|
out.append({**r, "models": models, "fit_level": worst})
|
|
return {"recipes": out, "sys_ram_gb": round(ram_gb, 1)}
|
|
|
|
|
|
@router.post("/install-recipe")
|
|
def install_recipe(req: InstallRecipeReq):
|
|
"""Komplettes Setup installieren: jedes Modell als Download-Job starten UND sofort mit
|
|
optimalem (gedeckeltem) Kontext in die config.yaml eintragen. llama-swap (-watch-config)
|
|
übernimmt es, sobald die Datei da ist."""
|
|
recipe = next((r for r in RECIPES if r["id"] == req.recipe_id), None)
|
|
if not recipe:
|
|
raise HTTPException(404, "Setup nicht gefunden.")
|
|
ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
env = {"HF_XET_HIGH_PERFORMANCE": "1"}
|
|
if req.hf_token:
|
|
env["HF_TOKEN"] = req.hf_token
|
|
cfg = read_config()
|
|
job_ids = []
|
|
for m in recipe["models"]:
|
|
file = _pick_gguf(m["repo"], m.get("quant", "Q4_K_M"))
|
|
if not file:
|
|
continue # kein GGUF im Repo gefunden -> Modell ueberspringen (Rest installiert trotzdem)
|
|
target = MODELS_DIR / m["repo"].split("/")[-1]
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
args = ["hf", "download", m["repo"], file, "--local-dir", str(target)]
|
|
jid = start_job(args, f"download {m['name']}", env=env)
|
|
JOBS[jid]["result_path"] = str(target / file)
|
|
job_ids.append(jid)
|
|
# Eintrag jetzt schon schreiben — optimaler Kontext, aber gedeckelt fuer schnellen Erststart.
|
|
ctx = min(max_ctx_for(m["params_b"], m["quant"], ram_gb), 32768)
|
|
path = str(target / file)
|
|
cmd = CMD_TEMPLATE.replace("{model}", path).replace("{ctx}", str(ctx))
|
|
cfg["models"][m["role"]] = {"cmd": LiteralScalarString(cmd + "\n"), "ttl": DEFAULT_TTL}
|
|
write_config(cfg)
|
|
return {"job_ids": job_ids, "count": len(job_ids)}
|
|
|
|
|
|
def _pick_gguf(repo: str, quant: str = "Q4_K_M") -> str | None:
|
|
"""Beste GGUF-Datei eines Repos auflösen: bevorzugt gewünschten Quant, keine Split-Teile."""
|
|
try:
|
|
with httpx.Client(timeout=10.0) as c:
|
|
tree = c.get(f"https://huggingface.co/api/models/{repo}/tree/main").json()
|
|
except Exception: # noqa: BLE001
|
|
return None
|
|
ggufs = [f["path"] for f in tree if isinstance(f, dict) and str(f.get("path", "")).endswith(".gguf")]
|
|
if not ggufs:
|
|
return None
|
|
pref = [g for g in ggufs if quant.lower() in g.lower() and "-of-" not in g]
|
|
nosplit = [g for g in ggufs if "-of-" not in g]
|
|
return (pref or nosplit or ggufs)[0]
|
|
|
|
|
|
@router.get("/upgrades")
|
|
def upgrades():
|
|
"""Schlägt für installierte 'alte' Modelle ein aktuelleres vor (UPGRADES-Map)."""
|
|
ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
installed = (read_config().get("models") or {})
|
|
out, seen = [], set()
|
|
for up in UPGRADES:
|
|
new_base = up["repo"].split("/")[-1].lower()
|
|
if any(new_base in str(s.get("cmd", "")).lower() for s in installed.values()):
|
|
continue # neueres Modell schon installiert
|
|
old = None
|
|
for alias, spec in installed.items():
|
|
hay = (alias + " " + str(spec.get("cmd", ""))).lower()
|
|
if any(k in hay for k in up["match"]):
|
|
old = alias
|
|
break
|
|
if not old or up["repo"] in seen:
|
|
continue
|
|
seen.add(up["repo"])
|
|
out.append({
|
|
"name": up["name"], "repo": up["repo"], "params_b": up["params_b"], "quant": up["quant"],
|
|
"why": up["why"], "old": old, "role": old,
|
|
"fit": evaluate_fit(up["params_b"], up["quant"], 8192, ram_gb),
|
|
"optimal_ctx": max_ctx_for(up["params_b"], up["quant"], ram_gb),
|
|
})
|
|
return {"upgrades": out}
|
|
|
|
|
|
@router.post("/install-model")
|
|
def install_model(req: InstallModelReq):
|
|
"""Ein einzelnes Modell installieren (Download + Einpflegen unter 'role', optimaler ctx)."""
|
|
file = _pick_gguf(req.repo, req.quant)
|
|
if not file:
|
|
raise HTTPException(404, "Keine GGUF-Datei im Repo gefunden.")
|
|
ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
env = {"HF_XET_HIGH_PERFORMANCE": "1"}
|
|
if req.hf_token:
|
|
env["HF_TOKEN"] = req.hf_token
|
|
target = MODELS_DIR / req.repo.split("/")[-1]
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
jid = start_job(["hf", "download", req.repo, file, "--local-dir", str(target)],
|
|
f"download {req.repo.split('/')[-1]}", env=env)
|
|
JOBS[jid]["result_path"] = str(target / file)
|
|
cfg = read_config()
|
|
ctx = min(max_ctx_for(req.params_b, req.quant, ram_gb), 32768)
|
|
cmd = CMD_TEMPLATE.replace("{model}", str(target / file)).replace("{ctx}", str(ctx))
|
|
cfg["models"][req.role] = {"cmd": LiteralScalarString(cmd + "\n"), "ttl": DEFAULT_TTL}
|
|
write_config(cfg)
|
|
return {"job_id": jid}
|
|
|