Hitonabi/mission-control-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement prompt caching, speculative decoding config, parallel slots, and dynamic pricing metrics

Browse Source
This commit is contained in:
Hitonabi
2026-06-26 14:00:32 +02:00
parent 2c60caf790
commit 35dcc69ba5
12 changed files with 216 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/config.py
+1 -1
View File
@@ -24,7 +24,7 @@ MEMORY_DB = Path(os.environ.get("MC_MEMORY_DB", str(MODELS_DIR / "mc2-memory.db"
# Befehl-Vorlage für llama-swap: {model}=GGUF-Pfad, {ctx}=Kontext, ${PORT} bleibt stehen.
_DEFAULT_CMD_TEMPLATE = (
"llama-server -m {model} --host 127.0.0.1 --port ${PORT} "
"-c {ctx} -ngl 999 -fa 1 --no-mmap"
"-c {ctx} -ngl 999 -fa 1 --no-mmap --prompt-cache --prompt-cache-all"
)
CMD_TEMPLATE = os.environ.get("MC_CMD_TEMPLATE", _DEFAULT_CMD_TEMPLATE)
if "{model}" not in CMD_TEMPLATE:
backend/migrate_config.py
+56
View File
@@ -0,0 +1,56 @@
import sys
from pathlib import Path
# Add backend directory to sys.path so we can import services
sys.path.append(str(Path(__file__).resolve().parent))
from services.llamaswap import read_config, write_config
from config import CONFIG_PATH
def migrate():
print(f"Reading config from {CONFIG_PATH}...")
if not CONFIG_PATH.exists():
print(f"Config path {CONFIG_PATH} does not exist. Skipping.")
return
cfg = read_config()
models = cfg.get("models", {})
draft_path = "/srv/models/drafts/qwen2.5-1.5b-instruct-q4_k_m.gguf"
for name, spec in models.items():
if not isinstance(spec, dict):
continue
cmd = spec.get("cmd", "")
if not cmd:
continue
print(f"Migrating model: {name}")
# 1. Ensure prompt caching flags exist
if "--prompt-cache " not in cmd and not cmd.endswith("--prompt-cache") and not cmd.endswith("--prompt-cache\n"):
cmd = cmd.strip() + " --prompt-cache"
if "--prompt-cache-all" not in cmd:
cmd = cmd.strip() + " --prompt-cache-all"
# 2. Extract aliases/role
aliases = spec.get("aliases", [])
role = aliases[0] if aliases else None
# 3. Add parallel and speculative decoding for fast and coder
if role in ("fast", "coder"):
if "--parallel" not in cmd:
cmd = cmd.strip() + " --parallel 2"
if "--spec-draft-model" not in cmd:
cmd = cmd.strip() + f" --spec-draft-model {draft_path}"
# Update cmd
from ruamel.yaml.scalarstring import LiteralScalarString
spec["cmd"] = LiteralScalarString(cmd.strip() + "\n")
print(f"Writing updated config back to {CONFIG_PATH}...")
write_config(cfg)
print("Migration completed successfully!")
if __name__ == "__main__":
migrate()
backend/routers/gateway_proxy.py
+2 -2
View File
@@ -52,7 +52,7 @@ async def _proxy(path: str, request: Request):
prompt = usage.get("prompt_tokens", 0)
completion = usage.get("completion_tokens", 0)
if prompt or completion:
increment_tokens(prompt, completion)
increment_tokens(prompt, completion, model=alias)
except Exception:
pass
except Exception:
@@ -69,7 +69,7 @@ async def _proxy(path: str, request: Request):
prompt = usage.get("prompt_tokens", 0)
completion = usage.get("completion_tokens", 0)
if prompt or completion:
increment_tokens(prompt, completion)
increment_tokens(prompt, completion, model=alias)
except Exception:
pass
return JSONResponse(resp_json, status_code=r.status_code, headers=routed)
backend/routers/system.py
+43 -3
View File
@@ -91,6 +91,7 @@ def self_update() -> dict:
from services.token_stats import get_stats
from services.llamaswap import list_models
@router.get("/system/token-stats")
def token_stats() -> dict:
@@ -99,9 +100,48 @@ def token_stats() -> dict:
c = stats.get("completion_tokens", 0)
total = p + c
# Juni 2026 API-Preise (deutlich gestiegen):
# Premium-Modelle (Claude 4 / GPT-5): 15,00 $ / 1M Input-Tokens und 75,00 $ / 1M Output-Tokens
saved_usd = (p * 15.0 + c * 75.0) / 1_000_000.0
# Map model IDs and aliases to their respective roles for pricing resolution
role_map = {}
try:
for m in list_models():
role_map[m["name"].lower()] = m.get("role")
for alias in m.get("aliases", []):
role_map[alias.lower()] = m.get("role")
except Exception:
pass
# Dynamic pricing tiers based on model class in June 2026
PRICING = {
"heavy": (15.0, 75.0),
"coder": (3.0, 15.0),
"hermes": (1.0, 5.0),
"fast": (0.15, 0.60),
"scout": (0.15, 0.60),
"vision": (0.15, 0.60),
"reasoning": (0.15, 0.60),
}
modeled_p = 0
modeled_c = 0
saved_usd = 0.0
models_data = stats.get("models") or {}
for m_name, m_tokens in models_data.items():
mp = m_tokens.get("prompt", 0)
mc = m_tokens.get("completion", 0)
modeled_p += mp
modeled_c += mc
role = role_map.get(m_name, m_name)
rate_in, rate_out = PRICING.get(role, (0.15, 0.60))
saved_usd += (mp * rate_in + mc * rate_out) / 1_000_000.0
# Baseline/legacy tokens calculated at premium rates ($15.00 / $75.00)
# to preserve historical savings value prior to model-specific logging
baseline_p = max(0, p - modeled_p)
baseline_c = max(0, c - modeled_c)
saved_usd += (baseline_p * 15.0 + baseline_c * 75.0) / 1_000_000.0
saved_eur = saved_usd * 0.92 # 1 USD = 0.92 EUR
return {
backend/services/llamaswap.py
+21
View File
@@ -57,6 +57,15 @@ def _parse_model(name: str, spec: dict) -> dict:
aliases = [str(a) for a in aliases]
role = aliases[0].lower() if aliases else (name.lower() if name.lower() in ROLE_IDS else None)
prompt_cache = "--prompt-cache " in cmd or cmd.endswith("--prompt-cache") or "--prompt-cache-all" in cmd
spec_draft = None
if "--spec-draft-model" in cmd:
m_draft = re.search(r"--spec-draft-model\s+([^\s]+)", cmd)
if m_draft:
spec_draft = os.path.basename(m_draft.group(1).replace("'", "").replace('"', ""))
parallel_match = re.search(r"--parallel\s+(\d+)", cmd)
parallel_slots = int(parallel_match.group(1)) if parallel_match else 1
from services.caps import capabilities
return {
"name": name,
@@ -71,6 +80,9 @@ def _parse_model(name: str, spec: dict) -> dict:
"quant": quant,
"size_bytes": size_bytes,
"incomplete": not path,
"prompt_cache": prompt_cache,
"spec_draft_model": spec_draft,
"parallel_slots": parallel_slots,
"capabilities": capabilities(
name=filename or name, cmd=cmd,
gguf_path=(path if (path and os.path.exists(path)) else ""),
@@ -171,6 +183,15 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
cmd = _augment_vision(cmd, model_path, mmproj_path)
if jinja and "--jinja" not in cmd:
cmd += " --jinja"
role_lower = (role or "").strip().lower()
if role_lower in ("fast", "coder"):
if "--parallel" not in cmd:
cmd += " --parallel 2"
draft_path = "/srv/models/drafts/qwen2.5-1.5b-instruct-q4_k_m.gguf"
if os.path.exists(draft_path) and "--spec-draft-model" not in cmd:
cmd += f" --spec-draft-model {draft_path}"
cfg.setdefault("models", {})[model_id] = {
"cmd": LiteralScalarString(cmd + "\n"),
"ttl": ttl if ttl is not None else DEFAULT_TTL,
backend/services/token_stats.py
+12 -2
View File
@@ -27,9 +27,11 @@ def get_stats() -> dict:
data["prompt_tokens"] = 0
if "completion_tokens" not in data:
data["completion_tokens"] = 0
if "models" not in data:
data["models"] = {}
return data
except Exception:
return {"prompt_tokens": 0, "completion_tokens": 0}
return {"prompt_tokens": 0, "completion_tokens": 0, "models": {}}
def save_stats(stats: dict):
try:
@@ -39,8 +41,16 @@ def save_stats(stats: dict):
except Exception:
pass
def increment_tokens(prompt: int, completion: int):
def increment_tokens(prompt: int, completion: int, model: str = None):
stats = get_stats()
stats["prompt_tokens"] += prompt
stats["completion_tokens"] += completion
if model:
model = model.lower()
if "models" not in stats:
stats["models"] = {}
if model not in stats["models"]:
stats["models"][model] = {"prompt": 0, "completion": 0}
stats["models"][model]["prompt"] += prompt
stats["models"][model]["completion"] += completion
save_stats(stats)