mission-control-v2/backend/services/discover.py

"""
Automatische Modell-Entdeckung ("aktuell beste Modelle"): fragt vertrauenswürdige
HF-Orgs live ab, kategorisiert per Stichwort, rankt nach Hardware-Fit + Beliebtheit
und cached. Portiert aus Mission Control v1 (cookbook.py-Discover).

Wichtig (Greenfield-Fix gegen v1): EIN gemeinsamer Ranking-Helfer `rank_runnable`
ist die Quelle der Wahrheit — sowohl die „beste Empfehlung" je Kategorie als auch
spätere Auto-Setups nutzen ihn, damit sie nie auseinanderlaufen.
"""

import json
import os
import time

import httpx

import logging

from config import DISCOVER_CACHE_PATH, DISCOVER_TTL
from services.caps import capabilities
from services.fit import evaluate_fit, extract_params_b, max_ctx_for
from services.sources import CATEGORIES, SKIP_TOKENS, TRUSTED_AUTHORS

log = logging.getLogger(__name__)

_FIT_ORDER = {"perfect": 0, "marginal": 1, "too_tight": 2}


def _categorize(repo_id: str) -> str:
    low = repo_id.lower()
    for cat in CATEGORIES:
        if any(k in low for k in cat["kw"]):
            return cat["role"]
    return "scout"


def _fetch_author_models(author: str) -> list:
    url = (f"https://huggingface.co/api/models?author={author}"
           f"&filter=gguf&sort=downloads&direction=-1&limit=40")
    try:
        with httpx.Client(timeout=12.0) as c:
            data = c.get(url).json()
        return data if isinstance(data, list) else []
    except Exception:
        log.debug("discover: Abfrage für Autor %s fehlgeschlagen", author, exc_info=True)
        return []


def rank_runnable(models: list[dict]) -> list[dict]:
    """EINE Quelle der Wahrheit fürs Ranking lauffähiger Modelle:
    bestes Fit-Level zuerst (perfect < marginal), bei Gleichstand meistgeladen.
    Zu große Modelle (too_tight) fliegen raus."""
    return sorted(
        [m for m in models if m["fit"]["level"] != "too_tight"],
        key=lambda m: (_FIT_ORDER[m["fit"]["level"]], -int(m.get("downloads") or 0)),
    )


def refresh_discover(ram_gb: float) -> dict:
    """Quellen live abfragen, kategorisieren, ranken, cachen. Wirft nur, wenn KEINE
    Quelle erreichbar war."""
    raw, seen, ok = [], set(), 0
    for author in TRUSTED_AUTHORS:
        models = _fetch_author_models(author)
        if models:
            ok += 1
        for m in models:
            rid = m.get("id")
            if not rid or rid in seen:
                continue
            seen.add(rid)
            raw.append(m)
    if ok == 0:
        raise RuntimeError("Keine Quelle erreichbar.")

    by_cat: dict[str, list] = {c["role"]: [] for c in CATEGORIES}
    for m in raw:
        rid = m["id"]
        low = rid.lower()
        if any(tok in low for tok in SKIP_TOKENS):
            continue
        role = _categorize(rid)
        params_b = extract_params_b(rid)
        quant = "Q4_K_M"  # Referenz-Quant für die Fit-Einschätzung
        fit = evaluate_fit(params_b, quant, 8192, ram_gb, name=rid)
        tags = [str(t) for t in (m.get("tags") or [])]
        by_cat[role].append({
            "name": rid.split("/")[-1], "author": rid.split("/")[0], "repo": rid,
            "role": role, "params_b": params_b, "quant": quant, "tags": tags,
            "downloads": int(m.get("downloads") or 0), "likes": int(m.get("likes") or 0),
            "lastModified": m.get("lastModified"),
            "fit": fit, "optimal_ctx": max_ctx_for(params_b, quant, ram_gb),
            "caps": capabilities(name=rid, hf={"tags": tags}),
        })

    cats = []
    for c in CATEGORIES:
        items = by_cat[c["role"]]
        ranked = rank_runnable(items)
        # Top 4 je Kategorie (für die Anzeige) — gerankt, dann nach Downloads aufgefüllt.
        items.sort(key=lambda x: (x["fit"]["level"] != "too_tight", x["downloads"]), reverse=True)
        top = items[:4]
        if top:
            cats.append({
                "role": c["role"], "title": c["title"], "icon": c["icon"],
                "models": top,
                "recommended": ranked[0]["repo"] if ranked else None,
            })

    data = {"updated": time.time(), "categories": cats}
    try:
        DISCOVER_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
        tmp = DISCOVER_CACHE_PATH.with_name(DISCOVER_CACHE_PATH.name + ".tmp")
        tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
        os.replace(tmp, DISCOVER_CACHE_PATH)
    except Exception:
        log.debug("discover: Cache-Schreiben fehlgeschlagen (nur Beschleunigung)", exc_info=True)
    return data


def load_discover() -> dict | None:
    try:
        if DISCOVER_CACHE_PATH.exists():
            return json.loads(DISCOVER_CACHE_PATH.read_text(encoding="utf-8"))
    except Exception:
        log.debug("discover: Cache-Lesen fehlgeschlagen", exc_info=True)
    return None


def safe_discover(ram_gb: float) -> dict | None:
    """Aus Cache (wenn frisch) oder live; wirft nie — None wenn nichts da."""
    cached = load_discover()
    if cached and (time.time() - cached.get("updated", 0) < DISCOVER_TTL):
        return cached
    try:
        return refresh_discover(ram_gb)
    except Exception:
        log.warning("discover: Live-Refresh fehlgeschlagen, nutze Cache", exc_info=True)
        return cached