From d1ea505b7ae423b0b0d8720a075e294644d8036f Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Sun, 28 Jun 2026 22:18:32 +0200 Subject: [PATCH] =?UTF-8?q?Feat:=20Bildschirm-Sicht=20=E2=80=94=20/api/voi?= =?UTF-8?q?ce/chat=20akzeptiert=20optionales=20Bild?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChatIn.image (data:-URL); bei Bild wird die User-Message multimodal ([text]+[image_url]) an Hermes gebaut -> fast/Qwen3.6 (mmproj) bzw. Vision- Modell verarbeitet den Screenshot. Lucys 'Augen' fuer die Desktop-App. Co-Authored-By: Claude Opus 4.8 --- backend/routers/voice.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/routers/voice.py b/backend/routers/voice.py index a8b6005..63eeed6 100644 --- a/backend/routers/voice.py +++ b/backend/routers/voice.py @@ -38,6 +38,7 @@ class ChatIn(BaseModel): session_key: str = "" # optional: Langzeit-Memory-Scope system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen") model: str = "" + image: str = "" # optionale Bildschirm-Sicht: Bild als data:-URL (Lucys „Augen") @router.get("/voice/health") @@ -139,7 +140,15 @@ async def voice_chat(body: ChatIn) -> StreamingResponse: messages = [] if body.system: messages.append({"role": "system", "content": body.system}) - messages.append({"role": "user", "content": body.text}) + if body.image: + # Bildschirm-Sicht: User-Message als multimodal (Text + Bild). Das Agent-Modell (fast/Qwen3.6 + # mit mmproj) bzw. ein Vision-Modell verarbeitet das Bild via OpenAI-image_url. + messages.append({"role": "user", "content": [ + {"type": "text", "text": body.text}, + {"type": "image_url", "image_url": {"url": body.image}}, + ]}) + else: + messages.append({"role": "user", "content": body.text}) payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True} headers = { "Authorization": f"Bearer {HERMES_API_KEY}",