diff --git a/backend/routers/voice.py b/backend/routers/voice.py index a8b6005..63eeed6 100644 --- a/backend/routers/voice.py +++ b/backend/routers/voice.py @@ -38,6 +38,7 @@ class ChatIn(BaseModel): session_key: str = "" # optional: Langzeit-Memory-Scope system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen") model: str = "" + image: str = "" # optionale Bildschirm-Sicht: Bild als data:-URL (Lucys „Augen") @router.get("/voice/health") @@ -139,7 +140,15 @@ async def voice_chat(body: ChatIn) -> StreamingResponse: messages = [] if body.system: messages.append({"role": "system", "content": body.system}) - messages.append({"role": "user", "content": body.text}) + if body.image: + # Bildschirm-Sicht: User-Message als multimodal (Text + Bild). Das Agent-Modell (fast/Qwen3.6 + # mit mmproj) bzw. ein Vision-Modell verarbeitet das Bild via OpenAI-image_url. + messages.append({"role": "user", "content": [ + {"type": "text", "text": body.text}, + {"type": "image_url", "image_url": {"url": body.image}}, + ]}) + else: + messages.append({"role": "user", "content": body.text}) payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True} headers = { "Authorization": f"Bearer {HERMES_API_KEY}",