fix(v9): Hermes-Proxy non-streaming (Tool-Calls wurden im Stream geleakt)
Der Hermes-API-Server fuehrt im Streaming-Modus Tool-Calls nicht aus und streamt rohe <function=…></tool_call>-Tokens als Text. Non-streaming durchlaeuft den vollen Agent-Loop (Tools, Gedaechtnis). Proxy holt die fertige Antwort und streamt sie wortweise selbst ans Frontend. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+22
-36
@@ -145,8 +145,11 @@ async def hermes_chat(websocket: WebSocket):
|
|||||||
async def _proxy_chat(message: str, websocket: WebSocket) -> None:
|
async def _proxy_chat(message: str, websocket: WebSocket) -> None:
|
||||||
"""Eine Antwort vom Hermes-Agent-Server holen und Tokens ans WS relayen.
|
"""Eine Antwort vom Hermes-Agent-Server holen und Tokens ans WS relayen.
|
||||||
|
|
||||||
Unterstuetzt sowohl SSE-Streaming als auch eine nicht-streamende
|
WICHTIG: bewusst **non-streaming**. Im Streaming-Modus (`stream:true`) gibt
|
||||||
JSON-Antwort — in beiden Faellen genau EIN Agent-Lauf (kein Doppel-Call).
|
der Hermes-API-Server rohe Modell-Tokens aus und fuehrt Tool-Calls NICHT aus
|
||||||
|
(`<function=…></tool_call>` leakt in den Text). Non-streaming durchlaeuft den
|
||||||
|
vollen Agent-Loop (Tools, Gedaechtnis, Mehr-Schritt) und liefert die fertige
|
||||||
|
Antwort. Fuer fluessige Optik streamen wir sie wortweise selbst ans Frontend.
|
||||||
"""
|
"""
|
||||||
headers = {"Content-Type": "application/json"}
|
headers = {"Content-Type": "application/json"}
|
||||||
if HERMES_API_KEY:
|
if HERMES_API_KEY:
|
||||||
@@ -154,44 +157,27 @@ async def _proxy_chat(message: str, websocket: WebSocket) -> None:
|
|||||||
body = {
|
body = {
|
||||||
"model": "hermes-agent",
|
"model": "hermes-agent",
|
||||||
"messages": [{"role": "user", "content": message}],
|
"messages": [{"role": "user", "content": message}],
|
||||||
"stream": True,
|
|
||||||
}
|
}
|
||||||
timeout = httpx.Timeout(300.0, connect=10.0)
|
timeout = httpx.Timeout(300.0, connect=10.0)
|
||||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||||
async with client.stream(
|
r = await client.post(
|
||||||
"POST", f"{HERMES_API_URL}/chat/completions", headers=headers, json=body
|
f"{HERMES_API_URL}/chat/completions", headers=headers, json=body
|
||||||
) as resp:
|
)
|
||||||
if resp.status_code != 200:
|
if r.status_code != 200:
|
||||||
detail = (await resp.aread()).decode(errors="replace")[:300]
|
await websocket.send_json(
|
||||||
await websocket.send_json(
|
{"type": "error", "content": f"HTTP {r.status_code}: {r.text[:300]}"}
|
||||||
{"type": "error", "content": f"HTTP {resp.status_code}: {detail}"}
|
)
|
||||||
)
|
return
|
||||||
return
|
try:
|
||||||
|
content = r.json()["choices"][0]["message"].get("content", "") or ""
|
||||||
|
except Exception:
|
||||||
|
content = r.text
|
||||||
|
|
||||||
if "text/event-stream" in resp.headers.get("content-type", ""):
|
words = content.split(" ")
|
||||||
async for line in resp.aiter_lines():
|
for i, word in enumerate(words):
|
||||||
if not line.startswith("data:"):
|
token = word + (" " if i < len(words) - 1 else "")
|
||||||
continue
|
await websocket.send_json({"type": "token", "content": token})
|
||||||
data = line[5:].strip()
|
await asyncio.sleep(0.01)
|
||||||
if data == "[DONE]":
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
delta = json.loads(data)["choices"][0].get("delta", {})
|
|
||||||
piece = delta.get("content") or ""
|
|
||||||
except Exception:
|
|
||||||
piece = ""
|
|
||||||
if piece:
|
|
||||||
await websocket.send_json({"type": "token", "content": piece})
|
|
||||||
else:
|
|
||||||
# Server streamt nicht → ganze JSON-Antwort als ein Token senden
|
|
||||||
raw = await resp.aread()
|
|
||||||
try:
|
|
||||||
d = json.loads(raw)
|
|
||||||
content = d["choices"][0]["message"].get("content", "") or ""
|
|
||||||
except Exception:
|
|
||||||
content = raw.decode(errors="replace")
|
|
||||||
if content:
|
|
||||||
await websocket.send_json({"type": "token", "content": content})
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/hermes/transcribe", dependencies=[Depends(auth)])
|
@router.post("/hermes/transcribe", dependencies=[Depends(auth)])
|
||||||
|
|||||||
Reference in New Issue
Block a user