Backend: Box-Sync + Drift-Fixes (Review P0-3)
- deploy/llama-swap.config.yaml: Live-Box-Config jetzt versioniert (war untracked + veraltet) - config.py: Template -fa 1 -> -fa on (Box-Standard) - llamaswap.py: cache-reuse NICHT bei mmproj-Modellen anhaengen (verifizierte Vision-Falle) - voice.py: Thinking-Deaktivierung fuer Voice-Chat (MC_VOICE_NO_THINK) - connect.py: IDE-Snippets zeigen nur noch die coding-Lane Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
+1
-1
@@ -32,7 +32,7 @@ MEM0_SERVICE_URL = os.environ.get("MC_MEM0_SERVICE_URL", "http://127.0.0.1:8765"
|
||||
# macht llama-server ohnehin automatisch pro Slot (KV-Reuse).
|
||||
_DEFAULT_CMD_TEMPLATE = (
|
||||
"llama-server -m {model} --host 127.0.0.1 --port ${PORT} "
|
||||
"-c {ctx} -ngl 999 -fa 1 --no-mmap"
|
||||
"-c {ctx} -ngl 999 -fa on --no-mmap"
|
||||
)
|
||||
CMD_TEMPLATE = os.environ.get("MC_CMD_TEMPLATE", _DEFAULT_CMD_TEMPLATE)
|
||||
if "{model}" not in CMD_TEMPLATE:
|
||||
|
||||
@@ -208,6 +208,11 @@ async def voice_chat(body: ChatIn) -> StreamingResponse:
|
||||
user_text = f"[Bildschirm-Sicht — das ist gerade auf dem/den Schirm(en) zu sehen:\n{safe_desc}\n]\n\n{body.text}"
|
||||
messages.append({"role": "user", "content": user_text})
|
||||
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
||||
# Lucys Hirn (Qwen3.6) ist ein Thinking-Modell -> für die gesprochene Assistentin Thinking AUS,
|
||||
# sonst generiert es tausende Reasoning-Token VOR der kurzen Antwort (gemessen: 11k Token, ~30s TTFB).
|
||||
# Gleiches Muster wie die fast-Spur im Gateway (gateway_proxy.py) und die Mem0-Extraktion.
|
||||
if os.environ.get("MC_VOICE_NO_THINK", "1") not in ("0", "false", "False"):
|
||||
payload["chat_template_kwargs"] = {"enable_thinking": False}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {HERMES_API_KEY}",
|
||||
"X-Hermes-Session-Id": body.session_id,
|
||||
|
||||
+10
-10
@@ -15,9 +15,9 @@ from config import LLAMA_SWAP_URL, MEM0_SERVICE_URL, PORT
|
||||
|
||||
DEFAULT_HOST = "192.168.178.151"
|
||||
|
||||
# Modelle/Lanes, die der Gateway anbietet. Lanes zuerst: 'coding' (agentischer Coder) ist der
|
||||
# Standard für IDEs, 'chat' für Allgemeines; dahinter die direkten Aliase.
|
||||
GATEWAY_MODELS = ["coding", "chat", "fast", "heavy", "coder", "vision"]
|
||||
# IDEs bekommen NUR die 'coding'-Lane zu sehen — die Lane routet intern selbst auf das
|
||||
# passende Modell (Coder/Heavy/…). Ein einziger Eintrag, kein manuelles Modell-Wählen mehr.
|
||||
IDE_MODEL = "coding"
|
||||
|
||||
|
||||
def _gw(host: str) -> str:
|
||||
@@ -44,7 +44,7 @@ def build_snippets(host: str = DEFAULT_HOST,
|
||||
"npm": "@ai-sdk/openai-compatible",
|
||||
"name": "Bosgame Gateway",
|
||||
"options": {"baseURL": gw, "apiKey": "local"},
|
||||
"models": {m: {"name": m} for m in GATEWAY_MODELS},
|
||||
"models": {IDE_MODEL: {"name": IDE_MODEL}},
|
||||
}
|
||||
}
|
||||
}, indent=2)
|
||||
@@ -61,8 +61,8 @@ def build_snippets(host: str = DEFAULT_HOST,
|
||||
"bosgame": {
|
||||
"api_url": gw,
|
||||
"available_models": [
|
||||
{"name": m, "display_name": m, "max_tokens": 131072,
|
||||
"capabilities": {"tools": True}} for m in GATEWAY_MODELS
|
||||
{"name": IDE_MODEL, "display_name": IDE_MODEL, "max_tokens": 131072,
|
||||
"capabilities": {"tools": True}}
|
||||
],
|
||||
}
|
||||
}
|
||||
@@ -70,15 +70,15 @@ def build_snippets(host: str = DEFAULT_HOST,
|
||||
"assistant": {
|
||||
"default_model": {
|
||||
"provider": "openai_compatible",
|
||||
"model": "coding"
|
||||
"model": IDE_MODEL
|
||||
}
|
||||
}
|
||||
}, indent=2)
|
||||
|
||||
cont = json.dumps({
|
||||
"models": [
|
||||
{"title": f"Bosgame / {m}", "provider": "openai", "model": m,
|
||||
"apiBase": gw, "apiKey": "local"} for m in ("coding", "chat", "coder")
|
||||
{"title": f"Bosgame / {IDE_MODEL}", "provider": "openai", "model": IDE_MODEL,
|
||||
"apiBase": gw, "apiKey": "local"}
|
||||
]
|
||||
}, indent=2)
|
||||
|
||||
@@ -115,7 +115,7 @@ def build_snippets(host: str = DEFAULT_HOST,
|
||||
# Leitung 1 — das MODELL. Alle Snippets zeigen auf den OpenAI-kompatiblen Gateway.
|
||||
"tools": {
|
||||
"cline": {"label": "Roo Code / Cline", "lang": "json", "snippet": cline,
|
||||
"note": "OpenAI-Provider → Gateway. Lane 'coding' (agentischer Coder); 'chat' für Allgemeines."},
|
||||
"note": "OpenAI-Provider → Gateway. Nur Lane 'coding' — die Box routet intern selbst."},
|
||||
"cursor": {"label": "Cursor", "lang": "json", "snippet": cursor,
|
||||
"note": "Einstellungen ➔ Models ➔ OpenAI API key + Base URL."},
|
||||
"opencode": {"label": "OpenCode", "lang": "jsonc", "snippet": opencode,
|
||||
|
||||
@@ -221,7 +221,9 @@ def register_model(model_path: str, role: str | None = None, ctx: int = 8192,
|
||||
# KV-Cache-Reuse über Turns (Prompt-Cache wiederverwenden) — hilft allen Chat-Modellen
|
||||
# (Agent-Hirn, Coding, Multi-Turn). Spiegelt die auf der Box bewährten Flags wider, damit
|
||||
# neu installierte Modelle nicht hinter dem hand-getunten Stand zurückbleiben (Drift-Fix).
|
||||
if "--cache-reuse" not in cmd:
|
||||
# NICHT bei Vision-Modellen: --cache-reuse + --mmproj bricht llama-server (live verifiziert,
|
||||
# deshalb fahren vision/scout auf der Box ohne cache-reuse).
|
||||
if "--cache-reuse" not in cmd and "--mmproj" not in cmd:
|
||||
cmd += " --cache-reuse 256 -cram 16384"
|
||||
# IDE-Coding profitiert von Nebenläufigkeit; sonst Default 1 Slot = voller Kontext/Anfrage
|
||||
# (--parallel teilt den Kontext HART auf die Slots auf, s. docs/OPTIMIZATION_PLAN.md §9.4 V6).
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# Auto-generiert von provision.sh - per Mission Control erweiterbar
|
||||
healthCheckTimeout: 300
|
||||
globalTTL: 0
|
||||
|
||||
models:
|
||||
Qwen3.6-35B-A3B:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 65536 -ngl 999 -fa on --no-mmap --jinja --parallel 1 -cram 16384 --spec-type draft-mtp --spec-draft-n-max 3
|
||||
ttl: 0
|
||||
aliases:
|
||||
- hermes
|
||||
- fast
|
||||
Qwen3.5-122B-A10B:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3.5-122B-A10B-GGUF/Q4_K_M/Qwen3.5-122B-A10B-Q4_K_M-00001-of-00003.gguf --host 127.0.0.1 --port ${PORT} -c 32768 -ngl 999 -fa on --no-mmap --jinja --cache-reuse 256 -cram 16384
|
||||
ttl: 600
|
||||
aliases:
|
||||
- heavy
|
||||
Qwen3-Coder-Next:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3-Coder-Next-GGUF/Qwen3-Coder-Next-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 131072 -ngl 999 -fa on --no-mmap --jinja --cache-reuse 256 -cram 16384
|
||||
ttl: 600
|
||||
aliases:
|
||||
- coder
|
||||
Qwen3-VL-8B-Instruct:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3-VL-8B-Instruct-GGUF/Qwen3VL-8B-Instruct-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 32768 -ngl 999 -fa on --no-mmap --mmproj /srv/models/Qwen3-VL-8B-Instruct-GGUF/mmproj-Qwen3VL-8B-Instruct-F16.gguf --jinja
|
||||
ttl: 300
|
||||
aliases:
|
||||
- vision
|
||||
Qwen3-Embedding-0.6B:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf --host 127.0.0.1 --port ${PORT} --embedding --pooling last -ngl 999 -fa on --no-mmap -c 8192
|
||||
ttl: 0
|
||||
aliases:
|
||||
- embed
|
||||
Qwen3-Coder-30B-A3B-Instruct:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/Qwen3-Coder-30B-A3B-Instruct-GGUF/Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 131072 -ngl 999 -fa on --no-mmap --jinja --cache-reuse 256 -cram 16384
|
||||
ttl: 300
|
||||
aliases:
|
||||
- coder-lite
|
||||
GLM-4.6V-Flash:
|
||||
cmd: |
|
||||
llama-server -m /srv/models/GLM-4.6V-Flash-GGUF/GLM-4.6V-Flash-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 131072 -ngl 999 -fa on --no-mmap --mmproj /srv/models/GLM-4.6V-Flash-GGUF/mmproj-BF16.gguf --jinja
|
||||
ttl: 300
|
||||
aliases:
|
||||
- scout
|
||||
groups:
|
||||
brains:
|
||||
swap: false
|
||||
persist: true
|
||||
members:
|
||||
- Qwen3-Embedding-0.6B
|
||||
- Qwen3-VL-8B-Instruct
|
||||
- Qwen3.6-35B-A3B
|
||||
Reference in New Issue
Block a user