Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
+2
-1
@@ -18,7 +18,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
from starlette.requests import Request
|
||||
|
||||
from config import FRONTEND_DIST, VERSION
|
||||
from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system
|
||||
from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system, voice
|
||||
from services import warmer
|
||||
|
||||
# Zentrales Logging — Level via MC_LOG_LEVEL (INFO default). Eine Konfiguration
|
||||
@@ -68,6 +68,7 @@ app.include_router(system.router)
|
||||
app.include_router(connect.router)
|
||||
app.include_router(memory.router)
|
||||
app.include_router(agent.router)
|
||||
app.include_router(voice.router) # Sprache: STT/TTS-Proxy + Hermes-Agent-Chat (Voice-Tab)
|
||||
app.include_router(gateway_proxy.router) # OpenAI-kompatibler /v1-Gateway (model:auto)
|
||||
app.include_router(maintenance.router)
|
||||
|
||||
|
||||
@@ -60,6 +60,35 @@ GATEWAY_URL = os.environ.get("MC_GATEWAY_URL", f"http://127.0.0.1:{os.environ.ge
|
||||
# --- Hermes Agent (eigener Dienst auf der Box) -------------------------------
|
||||
# Gateway (OpenAI-API des Agenten) + interaktives Web-Terminal (ttyd → `hermes chat`).
|
||||
HERMES_API_URL = os.environ.get("HERMES_API_URL", "http://127.0.0.1:8642").rstrip("/")
|
||||
# API-Key der Hermes-`api_server`-Plattform (~/.hermes/.env: API_SERVER_KEY). Nötig für
|
||||
# /v1/chat/completions (Voice-Pipeline) — Bearer-Auth, sonst 401. Derselbe volle Agent
|
||||
# (Tools + geteiltes Mem0) wie CLI/Telegram, nur über HTTP.
|
||||
def _read_hermes_env(key: str) -> str:
|
||||
"""Liest einen Schlüssel aus ~/.hermes/.env (Fallback, falls nicht in der Prozess-Env).
|
||||
Der MC2-Dienst erbt die Hermes-Secrets sonst nicht."""
|
||||
try:
|
||||
env_path = Path(os.path.expanduser(os.environ.get("HERMES_HOME", "~/.hermes"))) / ".env"
|
||||
for line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith(f"{key}="):
|
||||
return line.split("=", 1)[1].strip().strip('"').strip("'")
|
||||
except OSError:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
HERMES_API_KEY = (
|
||||
os.environ.get("HERMES_API_KEY")
|
||||
or os.environ.get("API_SERVER_KEY")
|
||||
or _read_hermes_env("API_SERVER_KEY")
|
||||
)
|
||||
# Modellfeld im OpenAI-Request; die api_server-Plattform nutzt ihr konfiguriertes Hirn,
|
||||
# das Feld ist i.d.R. kosmetisch. Override via Env, falls die Plattform strikt prüft.
|
||||
HERMES_API_MODEL = os.environ.get("HERMES_API_MODEL", "hermes")
|
||||
|
||||
# --- Voice-Sidecar (STT faster-whisper + TTS Piper/Chatterbox) ---------------
|
||||
# Eigenes Python-3.12-venv (~/.voice/venv), analog Mem0-Sidecar. MC2 proxyt nach außen.
|
||||
VOICE_SERVICE_URL = os.environ.get("MC_VOICE_SERVICE_URL", "http://127.0.0.1:8650").rstrip("/")
|
||||
# Hermes-Terminal: ttyd-Web-Terminal der interaktiven Agent-CLI (Ersatz für AnythingLLM-Chat).
|
||||
# Wird in MC2 per iframe eingebettet (Terminal-Seite). Siehe deploy/hermes-terminal.service.
|
||||
HERMES_TERMINAL_URL = os.environ.get("MC_HERMES_TERMINAL_URL", "http://192.168.178.151:7681").rstrip("/")
|
||||
|
||||
@@ -14,7 +14,7 @@ from pydantic import BaseModel
|
||||
|
||||
import httpx
|
||||
|
||||
from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL
|
||||
from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL, VOICE_SERVICE_URL
|
||||
from services import backup as backup_svc
|
||||
from services.agent import agent_status
|
||||
from services.gateway import gateway_reachable
|
||||
@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
# Nur diese User-Dienste dürfen neugestartet werden.
|
||||
ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui"}
|
||||
ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui", "mem0-service", "voice-service"}
|
||||
# Quelle für Self-Update (auf der Box ~/mission-control-v2).
|
||||
SOURCE_DIR = os.path.expanduser(os.environ.get("MC2_SOURCE_DIR", "~/mission-control-v2"))
|
||||
|
||||
@@ -45,6 +45,13 @@ def _mem0_reachable() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _voice_reachable() -> bool:
|
||||
try:
|
||||
return httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=2).status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@router.get("/system/services")
|
||||
def services() -> dict:
|
||||
"""Aggregierte Erreichbarkeit aller Stack-Dienste (für die Health-Anzeige)."""
|
||||
@@ -57,6 +64,7 @@ def services() -> dict:
|
||||
{"name": "Hermes-Gateway", "url": HERMES_API_URL, "ok": a["gateway_reachable"]},
|
||||
{"name": "Hermes-Terminal", "url": a["terminal_url"], "ok": a["terminal_reachable"]},
|
||||
{"name": "Mem0 (Gedächtnis)", "url": MEM0_SERVICE_URL, "ok": _mem0_reachable()},
|
||||
{"name": "Voice (STT/TTS)", "url": VOICE_SERVICE_URL, "ok": _voice_reachable()},
|
||||
],
|
||||
"links": {
|
||||
"engine_ui": f"{LLAMA_SWAP_URL}/ui",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Voice-Endpoints für „Mit Hermes reden" (Browser-Voice + 3D-Avatar).
|
||||
|
||||
Dünner Layer: STT/TTS werden zum Voice-Sidecar (:8650) geproxyt; der Chat geht an den
|
||||
Hermes-`api_server` (:8642, OpenAI-kompatibel) — denselben vollen Agenten mit Tools +
|
||||
geteiltem Mem0 wie CLI/Telegram. Mit stabilem `X-Hermes-Session-Id` hält die Plattform den
|
||||
Transcript server-seitig, daher schickt der Client je Turn nur die neue User-Nachricht.
|
||||
|
||||
LAN-only (kein Token in der 2.0-Phase), wie die übrigen MC2-Endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from config import HERMES_API_KEY, HERMES_API_MODEL, HERMES_API_URL, VOICE_SERVICE_URL
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # Chatterbox-TTS auf CPU darf dauern
|
||||
|
||||
|
||||
class TTSIn(BaseModel):
|
||||
text: str
|
||||
engine: str = "piper"
|
||||
voice: str = ""
|
||||
language: str = ""
|
||||
ref_path: str = ""
|
||||
|
||||
|
||||
class ChatIn(BaseModel):
|
||||
text: str # die neue User-Äußerung (STT-Ergebnis)
|
||||
session_id: str # stabiler Voice-Faden → server-seitiger Transcript
|
||||
session_key: str = "" # optional: Langzeit-Memory-Scope
|
||||
system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen")
|
||||
model: str = ""
|
||||
|
||||
|
||||
@router.get("/voice/health")
|
||||
def voice_health() -> dict:
|
||||
"""Erreichbarkeit des Voice-Sidecars + ob der Hermes-API-Key gesetzt ist."""
|
||||
out: dict = {"sidecar": False, "hermes_key": bool(HERMES_API_KEY)}
|
||||
try:
|
||||
r = httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=httpx.Timeout(5.0))
|
||||
out["sidecar"] = r.status_code == 200
|
||||
out["detail"] = r.json() if r.status_code == 200 else None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
out["error"] = str(exc)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/voice/voices")
|
||||
def voice_voices() -> dict:
|
||||
try:
|
||||
r = httpx.get(f"{VOICE_SERVICE_URL}/voices", timeout=httpx.Timeout(10.0))
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(502, f"Voice-Sidecar nicht erreichbar: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/stt")
|
||||
async def voice_stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict:
|
||||
"""Mikro-Audio → Text (Proxy auf Sidecar /stt)."""
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
files = {"audio": (audio.filename or "rec.webm", data, audio.content_type or "audio/webm")}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(f"{VOICE_SERVICE_URL}/stt", files=files, data={"language": language})
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"STT fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/tts")
|
||||
async def voice_tts(body: TTSIn) -> Response:
|
||||
"""Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(f"{VOICE_SERVICE_URL}/tts", json=body.model_dump())
|
||||
r.raise_for_status()
|
||||
return Response(content=r.content, media_type=r.headers.get("content-type", "audio/wav"))
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"TTS fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/chat")
|
||||
async def voice_chat(body: ChatIn) -> StreamingResponse:
|
||||
"""Neue User-Äußerung → Hermes-Agent (api_server, streamend). SSE wird 1:1 durchgereicht.
|
||||
|
||||
Mit `X-Hermes-Session-Id` hält die Plattform den Verlauf — wir senden nur die neue Nachricht.
|
||||
Auth per Bearer (API_SERVER_KEY); ohne Key liefert :8642 ein 401."""
|
||||
if not HERMES_API_KEY:
|
||||
raise HTTPException(503, "HERMES_API_KEY/API_SERVER_KEY nicht gesetzt — Agent-Auth fehlt.")
|
||||
|
||||
messages = []
|
||||
if body.system:
|
||||
messages.append({"role": "system", "content": body.system})
|
||||
messages.append({"role": "user", "content": body.text})
|
||||
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {HERMES_API_KEY}",
|
||||
"X-Hermes-Session-Id": body.session_id,
|
||||
}
|
||||
if body.session_key:
|
||||
headers["X-Hermes-Session-Key"] = body.session_key
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(None, connect=5.0)) as client:
|
||||
async with client.stream(
|
||||
"POST", f"{HERMES_API_URL}/v1/chat/completions", json=payload, headers=headers,
|
||||
) as r:
|
||||
if r.status_code != 200:
|
||||
detail = (await r.aread()).decode("utf-8", "replace")[:500]
|
||||
yield f"data: {{\"error\": \"Hermes {r.status_code}: {detail}\"}}\n\n".encode()
|
||||
return
|
||||
async for chunk in r.aiter_raw():
|
||||
yield chunk
|
||||
except httpx.HTTPError as exc:
|
||||
yield f"data: {{\"error\": \"Verbindung zu Hermes fehlgeschlagen: {exc}\"}}\n\n".encode()
|
||||
|
||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||
@@ -19,7 +19,7 @@ from services import catalog, discover, jobengine, llamaswap, system
|
||||
|
||||
# System-Dienste (root, via sudo -n NOPASSWD) vs. User-Dienste (systemctl --user).
|
||||
SYSTEM_SERVICES = {"llama-swap"}
|
||||
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service"}
|
||||
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service", "voice-service"}
|
||||
|
||||
# Engine-Update: lädt den neuesten Vulkan-Build (deploy/update-engine.sh, läuft als root).
|
||||
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
|
||||
Reference in New Issue
Block a user