Hitonabi
68cad9c0f1
ChatIn.images (Liste, 1 data-URL je Monitor); _describe_images schickt alle Screenshots in EINER Nachricht ans VL-Modell -> Lucy sieht beide Bildschirme. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
207 lines
9.0 KiB
Python
207 lines
9.0 KiB
Python
"""
|
|
Voice-Endpoints für „Mit Hermes reden" (Browser-Voice + 3D-Avatar).
|
|
|
|
Dünner Layer: STT/TTS werden zum Voice-Sidecar (:8650) geproxyt; der Chat geht an den
|
|
Hermes-`api_server` (:8642, OpenAI-kompatibel) — denselben vollen Agenten mit Tools +
|
|
geteiltem Mem0 wie CLI/Telegram. Mit stabilem `X-Hermes-Session-Id` hält die Plattform den
|
|
Transcript server-seitig, daher schickt der Client je Turn nur die neue User-Nachricht.
|
|
|
|
LAN-only (kein Token in der 2.0-Phase), wie die übrigen MC2-Endpoints.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
|
|
import httpx
|
|
from fastapi import APIRouter, File, Form, HTTPException, UploadFile
|
|
from fastapi.responses import Response, StreamingResponse
|
|
from pydantic import BaseModel
|
|
|
|
from config import HERMES_API_KEY, HERMES_API_MODEL, HERMES_API_URL, LLAMA_SWAP_URL, VOICE_SERVICE_URL
|
|
|
|
log = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api")
|
|
|
|
# Bildschirm-Sicht: das DEDIZIERTE Vision-Modell (Qwen3-VL-8B) beschreibt das Bild; die Beschreibung
|
|
# geht als TEXT an Hermes -> Lucy behält ihr volles Hirn/Gedächtnis UND nutzt das bessere VL-Modell
|
|
# (statt der schwächeren Vision der fast-MoE). Per Env abschaltbar/umstellbar.
|
|
VISION_MODEL = os.environ.get("MC_VISION_MODEL", "vision")
|
|
|
|
|
|
async def _describe_images(image_urls: list[str], hint: str) -> str:
|
|
"""Lässt das Vision-Modell die Screenshots (1 je Monitor) knapp beschreiben (Deutsch).
|
|
Mehrere Bilder gehen in EINER Nachricht ans VL-Modell. Leerer String bei Fehler."""
|
|
multi = len(image_urls) > 1
|
|
intro = (f"Hier sind {len(image_urls)} Screenshots (je ein Monitor). Beschreibe knapp und präzise auf Deutsch, "
|
|
"was auf den Bildschirmen zu sehen ist (pro Monitor: App/Fenster, wichtige Inhalte, sichtbarer Text/Code). "
|
|
if multi else
|
|
"Beschreibe knapp und präzise auf Deutsch, was auf diesem Screenshot zu sehen ist "
|
|
"(App/Fenster, wichtige Inhalte, sichtbarer Text/Code). ")
|
|
content: list = [{"type": "text", "text": intro + "Frage des Nutzers dazu: " + hint}]
|
|
for u in image_urls:
|
|
content.append({"type": "image_url", "image_url": {"url": u}})
|
|
try:
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(120.0, connect=5.0)) as client:
|
|
r = await client.post(f"{LLAMA_SWAP_URL}/v1/chat/completions", json={
|
|
"model": VISION_MODEL, "max_tokens": 600, "stream": False,
|
|
"messages": [{"role": "user", "content": content}],
|
|
})
|
|
r.raise_for_status()
|
|
return (r.json().get("choices") or [{}])[0].get("message", {}).get("content", "").strip()
|
|
except Exception as exc:
|
|
log.warning("Vision-Beschreibung fehlgeschlagen: %s", exc)
|
|
return ""
|
|
|
|
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # Chatterbox-TTS auf CPU darf dauern
|
|
|
|
|
|
class TTSIn(BaseModel):
|
|
text: str
|
|
engine: str = "piper"
|
|
voice: str = ""
|
|
language: str = ""
|
|
ref_path: str = ""
|
|
|
|
|
|
class ChatIn(BaseModel):
|
|
text: str # die neue User-Äußerung (STT-Ergebnis)
|
|
session_id: str # stabiler Voice-Faden → server-seitiger Transcript
|
|
session_key: str = "" # optional: Langzeit-Memory-Scope
|
|
system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen")
|
|
model: str = ""
|
|
images: list[str] = [] # optionale Bildschirm-Sicht: ein data:-URL je Monitor (Lucys „Augen")
|
|
|
|
|
|
@router.get("/voice/health")
|
|
def voice_health() -> dict:
|
|
"""Erreichbarkeit des Voice-Sidecars + ob der Hermes-API-Key gesetzt ist."""
|
|
out: dict = {"sidecar": False, "hermes_key": bool(HERMES_API_KEY)}
|
|
try:
|
|
r = httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=httpx.Timeout(5.0))
|
|
out["sidecar"] = r.status_code == 200
|
|
out["detail"] = r.json() if r.status_code == 200 else None
|
|
except Exception as exc: # noqa: BLE001
|
|
out["error"] = str(exc)
|
|
return out
|
|
|
|
|
|
@router.get("/voice/voices")
|
|
def voice_voices() -> dict:
|
|
try:
|
|
r = httpx.get(f"{VOICE_SERVICE_URL}/voices", timeout=httpx.Timeout(10.0))
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(502, f"Voice-Sidecar nicht erreichbar: {exc}")
|
|
|
|
|
|
@router.post("/voice/stt")
|
|
async def voice_stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict:
|
|
"""Mikro-Audio → Text (Proxy auf Sidecar /stt)."""
|
|
data = await audio.read()
|
|
if not data:
|
|
raise HTTPException(400, "Leeres Audio.")
|
|
files = {"audio": (audio.filename or "rec.webm", data, audio.content_type or "audio/webm")}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
|
r = await client.post(f"{VOICE_SERVICE_URL}/stt", files=files, data={"language": language})
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(502, f"STT fehlgeschlagen: {exc}")
|
|
|
|
|
|
@router.post("/voice/reference")
|
|
async def voice_set_reference(audio: UploadFile = File(...)) -> dict:
|
|
"""Klon-Referenz (z.B. ElevenLabs-Erzeugnis) hochladen → Chatterbox nutzt sie. Proxy → Sidecar."""
|
|
data = await audio.read()
|
|
if not data:
|
|
raise HTTPException(400, "Leeres Audio.")
|
|
files = {"audio": (audio.filename or "ref.wav", data, audio.content_type or "audio/mpeg")}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
|
r = await client.post(f"{VOICE_SERVICE_URL}/reference", files=files)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(502, f"Referenz-Upload fehlgeschlagen: {exc}")
|
|
|
|
|
|
@router.get("/voice/reference")
|
|
def voice_get_reference() -> dict:
|
|
try:
|
|
r = httpx.get(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except Exception as exc: # noqa: BLE001
|
|
return {"active": False, "error": str(exc)}
|
|
|
|
|
|
@router.delete("/voice/reference")
|
|
def voice_clear_reference() -> dict:
|
|
try:
|
|
r = httpx.delete(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(502, f"Löschen fehlgeschlagen: {exc}")
|
|
|
|
|
|
@router.post("/voice/tts")
|
|
async def voice_tts(body: TTSIn) -> Response:
|
|
"""Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes."""
|
|
try:
|
|
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
|
r = await client.post(f"{VOICE_SERVICE_URL}/tts", json=body.model_dump())
|
|
r.raise_for_status()
|
|
return Response(content=r.content, media_type=r.headers.get("content-type", "audio/wav"))
|
|
except httpx.HTTPError as exc:
|
|
raise HTTPException(502, f"TTS fehlgeschlagen: {exc}")
|
|
|
|
|
|
@router.post("/voice/chat")
|
|
async def voice_chat(body: ChatIn) -> StreamingResponse:
|
|
"""Neue User-Äußerung → Hermes-Agent (api_server, streamend). SSE wird 1:1 durchgereicht.
|
|
|
|
Mit `X-Hermes-Session-Id` hält die Plattform den Verlauf — wir senden nur die neue Nachricht.
|
|
Auth per Bearer (API_SERVER_KEY); ohne Key liefert :8642 ein 401."""
|
|
if not HERMES_API_KEY:
|
|
raise HTTPException(503, "HERMES_API_KEY/API_SERVER_KEY nicht gesetzt — Agent-Auth fehlt.")
|
|
|
|
messages = []
|
|
if body.system:
|
|
messages.append({"role": "system", "content": body.system})
|
|
user_text = body.text
|
|
imgs = [u for u in (body.images or []) if u]
|
|
if imgs:
|
|
# Bildschirm-Sicht: erst das Vision-Modell die Monitore beschreiben lassen, dann die Beschreibung
|
|
# als TEXT-Kontext an Hermes (Lucy antwortet mit vollem Hirn/Gedächtnis, sieht via besserem VL-Modell).
|
|
desc = await _describe_images(imgs, body.text)
|
|
if desc:
|
|
user_text = f"[Bildschirm-Sicht — das ist gerade auf dem/den Schirm(en) zu sehen:\n{desc}\n]\n\n{body.text}"
|
|
messages.append({"role": "user", "content": user_text})
|
|
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
|
headers = {
|
|
"Authorization": f"Bearer {HERMES_API_KEY}",
|
|
"X-Hermes-Session-Id": body.session_id,
|
|
}
|
|
if body.session_key:
|
|
headers["X-Hermes-Session-Key"] = body.session_key
|
|
|
|
async def gen():
|
|
try:
|
|
async with httpx.AsyncClient(timeout=httpx.Timeout(None, connect=5.0)) as client:
|
|
async with client.stream(
|
|
"POST", f"{HERMES_API_URL}/v1/chat/completions", json=payload, headers=headers,
|
|
) as r:
|
|
if r.status_code != 200:
|
|
detail = (await r.aread()).decode("utf-8", "replace")[:500]
|
|
yield f"data: {{\"error\": \"Hermes {r.status_code}: {detail}\"}}\n\n".encode()
|
|
return
|
|
async for chunk in r.aiter_raw():
|
|
yield chunk
|
|
except httpx.HTTPError as exc:
|
|
yield f"data: {{\"error\": \"Verbindung zu Hermes fehlgeschlagen: {exc}\"}}\n\n".encode()
|
|
|
|
return StreamingResponse(gen(), media_type="text/event-stream")
|