Feat: Bildschirm-Sicht — /api/voice/chat akzeptiert optionales Bild
ChatIn.image (data:-URL); bei Bild wird die User-Message multimodal ([text]+[image_url]) an Hermes gebaut -> fast/Qwen3.6 (mmproj) bzw. Vision- Modell verarbeitet den Screenshot. Lucys 'Augen' fuer die Desktop-App. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -38,6 +38,7 @@ class ChatIn(BaseModel):
|
|||||||
session_key: str = "" # optional: Langzeit-Memory-Scope
|
session_key: str = "" # optional: Langzeit-Memory-Scope
|
||||||
system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen")
|
system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen")
|
||||||
model: str = ""
|
model: str = ""
|
||||||
|
image: str = "" # optionale Bildschirm-Sicht: Bild als data:-URL (Lucys „Augen")
|
||||||
|
|
||||||
|
|
||||||
@router.get("/voice/health")
|
@router.get("/voice/health")
|
||||||
@@ -139,7 +140,15 @@ async def voice_chat(body: ChatIn) -> StreamingResponse:
|
|||||||
messages = []
|
messages = []
|
||||||
if body.system:
|
if body.system:
|
||||||
messages.append({"role": "system", "content": body.system})
|
messages.append({"role": "system", "content": body.system})
|
||||||
messages.append({"role": "user", "content": body.text})
|
if body.image:
|
||||||
|
# Bildschirm-Sicht: User-Message als multimodal (Text + Bild). Das Agent-Modell (fast/Qwen3.6
|
||||||
|
# mit mmproj) bzw. ein Vision-Modell verarbeitet das Bild via OpenAI-image_url.
|
||||||
|
messages.append({"role": "user", "content": [
|
||||||
|
{"type": "text", "text": body.text},
|
||||||
|
{"type": "image_url", "image_url": {"url": body.image}},
|
||||||
|
]})
|
||||||
|
else:
|
||||||
|
messages.append({"role": "user", "content": body.text})
|
||||||
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
||||||
headers = {
|
headers = {
|
||||||
"Authorization": f"Bearer {HERMES_API_KEY}",
|
"Authorization": f"Bearer {HERMES_API_KEY}",
|
||||||
|
|||||||
Reference in New Issue
Block a user