From edf1614e66cd4963d8bc5d4bb2b93369c7fbf773 Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Tue, 7 Jul 2026 17:11:50 +0200 Subject: [PATCH] feat(perf): Optimize KV-Cache to Q4_K and fix Hermes Context bounds --- backend/routers/gateway_proxy.py | 14 +++++++++++++- deploy/llama-swap.config.yaml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/routers/gateway_proxy.py b/backend/routers/gateway_proxy.py index 485fe8a..d410988 100644 --- a/backend/routers/gateway_proxy.py +++ b/backend/routers/gateway_proxy.py @@ -98,11 +98,23 @@ async def _proxy(path: str, request: Request): client = request.app.state.gw_client # geteilter Keep-Alive-Client (siehe app.py lifespan) if body.get("stream"): + req = client.build_request("POST", url, json=body, timeout=None) + r = await client.send(req, stream=True) + if r.status_code != 200: + await r.aread() + try: + resp_json = r.json() + except Exception: + resp_json = {"error": {"message": r.text, "type": "upstream_error"}} + return JSONResponse(resp_json, status_code=r.status_code, headers=routed) + async def gen(): - async with client.stream("POST", url, json=body, timeout=None) as r: + try: async for chunk in r.aiter_raw(): record_stream_chunk(chunk, alias) yield chunk + finally: + await r.aclose() return StreamingResponse(gen(), media_type="text/event-stream", headers=routed) r = await client.post(url, json=body, timeout=600.0) diff --git a/deploy/llama-swap.config.yaml b/deploy/llama-swap.config.yaml index 31a0a3b..c30e5ec 100644 --- a/deploy/llama-swap.config.yaml +++ b/deploy/llama-swap.config.yaml @@ -8,7 +8,7 @@ models: # nicht mehr); KV Q8_0 macht das speicherneutral zu vorher (65k f16). Bench 2026-07-02: # Q8_0 kostet 0 t/s, MTP n-max 3 = +26% vs. ohne Spec. cmd: | - llama-server -m /srv/models/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 131072 -ngl 999 -fa on --no-mmap --jinja --parallel 2 -cram 16384 -ctk q8_0 -ctv q8_0 --spec-type draft-mtp --spec-draft-n-max 3 + llama-server -m /srv/models/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf --host 127.0.0.1 --port ${PORT} -c 131072 -ngl 999 -fa on --no-mmap --jinja --parallel 2 -cram 16384 -ctk q4_k -ctv q4_k --spec-type draft-mtp --spec-draft-n-max 3 ttl: 0 aliases: - hermes