Implement prompt caching, speculative decoding config, parallel slots, and dynamic pricing metrics

2026-06-26 14:00:32 +02:00
parent 2c60caf790
commit 35dcc69ba5
12 changed files with 216 additions and 35 deletions
@@ -0,0 +1,56 @@
+import sys
+from pathlib import Path
+
+# Add backend directory to sys.path so we can import services
+sys.path.append(str(Path(__file__).resolve().parent))
+
+from services.llamaswap import read_config, write_config
+from config import CONFIG_PATH
+
+def migrate():
+    print(f"Reading config from {CONFIG_PATH}...")
+    if not CONFIG_PATH.exists():
+        print(f"Config path {CONFIG_PATH} does not exist. Skipping.")
+        return
+        
+    cfg = read_config()
+    models = cfg.get("models", {})
+    
+    draft_path = "/srv/models/drafts/qwen2.5-1.5b-instruct-q4_k_m.gguf"
+    
+    for name, spec in models.items():
+        if not isinstance(spec, dict):
+            continue
+        cmd = spec.get("cmd", "")
+        if not cmd:
+            continue
+            
+        print(f"Migrating model: {name}")
+        
+        # 1. Ensure prompt caching flags exist
+        if "--prompt-cache " not in cmd and not cmd.endswith("--prompt-cache") and not cmd.endswith("--prompt-cache\n"):
+            cmd = cmd.strip() + " --prompt-cache"
+        if "--prompt-cache-all" not in cmd:
+            cmd = cmd.strip() + " --prompt-cache-all"
+            
+        # 2. Extract aliases/role
+        aliases = spec.get("aliases", [])
+        role = aliases[0] if aliases else None
+        
+        # 3. Add parallel and speculative decoding for fast and coder
+        if role in ("fast", "coder"):
+            if "--parallel" not in cmd:
+                cmd = cmd.strip() + " --parallel 2"
+            if "--spec-draft-model" not in cmd:
+                cmd = cmd.strip() + f" --spec-draft-model {draft_path}"
+                
+        # Update cmd
+        from ruamel.yaml.scalarstring import LiteralScalarString
+        spec["cmd"] = LiteralScalarString(cmd.strip() + "\n")
+        
+    print(f"Writing updated config back to {CONFIG_PATH}...")
+    write_config(cfg)
+    print("Migration completed successfully!")
+
+if __name__ == "__main__":
+    migrate()