fix(v9): Hermes-Proxy non-streaming (Tool-Calls wurden im Stream geleakt)
Der Hermes-API-Server fuehrt im Streaming-Modus Tool-Calls nicht aus und streamt rohe <function=…></tool_call>-Tokens als Text. Non-streaming durchlaeuft den vollen Agent-Loop (Tools, Gedaechtnis). Proxy holt die fertige Antwort und streamt sie wortweise selbst ans Frontend. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+22
-36
@@ -145,8 +145,11 @@ async def hermes_chat(websocket: WebSocket):
|
||||
async def _proxy_chat(message: str, websocket: WebSocket) -> None:
|
||||
"""Eine Antwort vom Hermes-Agent-Server holen und Tokens ans WS relayen.
|
||||
|
||||
Unterstuetzt sowohl SSE-Streaming als auch eine nicht-streamende
|
||||
JSON-Antwort — in beiden Faellen genau EIN Agent-Lauf (kein Doppel-Call).
|
||||
WICHTIG: bewusst **non-streaming**. Im Streaming-Modus (`stream:true`) gibt
|
||||
der Hermes-API-Server rohe Modell-Tokens aus und fuehrt Tool-Calls NICHT aus
|
||||
(`<function=…></tool_call>` leakt in den Text). Non-streaming durchlaeuft den
|
||||
vollen Agent-Loop (Tools, Gedaechtnis, Mehr-Schritt) und liefert die fertige
|
||||
Antwort. Fuer fluessige Optik streamen wir sie wortweise selbst ans Frontend.
|
||||
"""
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if HERMES_API_KEY:
|
||||
@@ -154,44 +157,27 @@ async def _proxy_chat(message: str, websocket: WebSocket) -> None:
|
||||
body = {
|
||||
"model": "hermes-agent",
|
||||
"messages": [{"role": "user", "content": message}],
|
||||
"stream": True,
|
||||
}
|
||||
timeout = httpx.Timeout(300.0, connect=10.0)
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
async with client.stream(
|
||||
"POST", f"{HERMES_API_URL}/chat/completions", headers=headers, json=body
|
||||
) as resp:
|
||||
if resp.status_code != 200:
|
||||
detail = (await resp.aread()).decode(errors="replace")[:300]
|
||||
await websocket.send_json(
|
||||
{"type": "error", "content": f"HTTP {resp.status_code}: {detail}"}
|
||||
)
|
||||
return
|
||||
r = await client.post(
|
||||
f"{HERMES_API_URL}/chat/completions", headers=headers, json=body
|
||||
)
|
||||
if r.status_code != 200:
|
||||
await websocket.send_json(
|
||||
{"type": "error", "content": f"HTTP {r.status_code}: {r.text[:300]}"}
|
||||
)
|
||||
return
|
||||
try:
|
||||
content = r.json()["choices"][0]["message"].get("content", "") or ""
|
||||
except Exception:
|
||||
content = r.text
|
||||
|
||||
if "text/event-stream" in resp.headers.get("content-type", ""):
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
delta = json.loads(data)["choices"][0].get("delta", {})
|
||||
piece = delta.get("content") or ""
|
||||
except Exception:
|
||||
piece = ""
|
||||
if piece:
|
||||
await websocket.send_json({"type": "token", "content": piece})
|
||||
else:
|
||||
# Server streamt nicht → ganze JSON-Antwort als ein Token senden
|
||||
raw = await resp.aread()
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
content = d["choices"][0]["message"].get("content", "") or ""
|
||||
except Exception:
|
||||
content = raw.decode(errors="replace")
|
||||
if content:
|
||||
await websocket.send_json({"type": "token", "content": content})
|
||||
words = content.split(" ")
|
||||
for i, word in enumerate(words):
|
||||
token = word + (" " if i < len(words) - 1 else "")
|
||||
await websocket.send_json({"type": "token", "content": token})
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
@router.post("/hermes/transcribe", dependencies=[Depends(auth)])
|
||||
|
||||
Reference in New Issue
Block a user