Implement prompt caching, speculative decoding config, parallel slots, and dynamic pricing metrics

2026-06-26 14:00:32 +02:00
parent 2c60caf790
commit 35dcc69ba5
12 changed files with 216 additions and 35 deletions
@@ -24,7 +24,7 @@ MEMORY_DB = Path(os.environ.get("MC_MEMORY_DB", str(MODELS_DIR / "mc2-memory.db"
 # Befehl-Vorlage für llama-swap: {model}=GGUF-Pfad, {ctx}=Kontext, ${PORT} bleibt stehen.
 _DEFAULT_CMD_TEMPLATE = (
    "llama-server -m {model} --host 127.0.0.1 --port ${PORT} "
-    "-c {ctx} -ngl 999 -fa 1 --no-mmap"
+    "-c {ctx} -ngl 999 -fa 1 --no-mmap --prompt-cache --prompt-cache-all"
 )
 CMD_TEMPLATE = os.environ.get("MC_CMD_TEMPLATE", _DEFAULT_CMD_TEMPLATE)
 if "{model}" not in CMD_TEMPLATE: