Implement prompt caching, speculative decoding config, parallel slots, and dynamic pricing metrics

backend/services/llamaswap.py

@@ -57,6 +57,15 @@ def _parse_model(name: str, spec: dict) -> dict:
     aliases = [str(a) for a in aliases]
     role = aliases[0].lower() if aliases else (name.lower() if name.lower() in ROLE_IDS else None)
 
+    prompt_cache = "--prompt-cache " in cmd or cmd.endswith("--prompt-cache") or "--prompt-cache-all" in cmd
+    spec_draft = None
+    if "--spec-draft-model" in cmd:
+        m_draft = re.search(r"--spec-draft-model\s+([^\s]+)", cmd)
+        if m_draft:
+            spec_draft = os.path.basename(m_draft.group(1).replace("'", "").replace('"', ""))
+    parallel_match = re.search(r"--parallel\s+(\d+)", cmd)
+    parallel_slots = int(parallel_match.group(1)) if parallel_match else 1
+
     from services.caps import capabilities
     return {
         "name": name,
@@ -71,6 +80,9 @@ def _parse_model(name: str, spec: dict) -> dict:
         "quant": quant,
         "size_bytes": size_bytes,
         "incomplete": not path,
+        "prompt_cache": prompt_cache,
+        "spec_draft_model": spec_draft,
+        "parallel_slots": parallel_slots,
         "capabilities": capabilities(
             name=filename or name, cmd=cmd,
             gguf_path=(path if (path and os.path.exists(path)) else ""),
@@ -171,6 +183,15 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
     cmd = _augment_vision(cmd, model_path, mmproj_path)
     if jinja and "--jinja" not in cmd:
         cmd += " --jinja"
+        
+    role_lower = (role or "").strip().lower()
+    if role_lower in ("fast", "coder"):
+        if "--parallel" not in cmd:
+            cmd += " --parallel 2"
+        draft_path = "/srv/models/drafts/qwen2.5-1.5b-instruct-q4_k_m.gguf"
+        if os.path.exists(draft_path) and "--spec-draft-model" not in cmd:
+            cmd += f" --spec-draft-model {draft_path}"
+            
     cfg.setdefault("models", {})[model_id] = {
         "cmd": LiteralScalarString(cmd + "\n"),
         "ttl": ttl if ttl is not None else DEFAULT_TTL,

backend/services/token_stats.py

@@ -27,9 +27,11 @@ def get_stats() -> dict:
                 data["prompt_tokens"] = 0
             if "completion_tokens" not in data:
                 data["completion_tokens"] = 0
+            if "models" not in data:
+                data["models"] = {}
             return data
     except Exception:
-        return {"prompt_tokens": 0, "completion_tokens": 0}
+        return {"prompt_tokens": 0, "completion_tokens": 0, "models": {}}
 
 def save_stats(stats: dict):
     try:
@@ -39,8 +41,16 @@ def save_stats(stats: dict):
     except Exception:
         pass
 
-def increment_tokens(prompt: int, completion: int):
+def increment_tokens(prompt: int, completion: int, model: str = None):
     stats = get_stats()
     stats["prompt_tokens"] += prompt
     stats["completion_tokens"] += completion
+    if model:
+        model = model.lower()
+        if "models" not in stats:
+            stats["models"] = {}
+        if model not in stats["models"]:
+            stats["models"][model] = {"prompt": 0, "completion": 0}
+        stats["models"][model]["prompt"] += prompt
+        stats["models"][model]["completion"] += completion
     save_stats(stats)