diff --git a/backend/app.py b/backend/app.py index a964cb8..723aaa5 100644 --- a/backend/app.py +++ b/backend/app.py @@ -18,7 +18,7 @@ from fastapi.staticfiles import StaticFiles from starlette.requests import Request from config import FRONTEND_DIST, VERSION -from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system +from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system, voice from services import warmer # Zentrales Logging — Level via MC_LOG_LEVEL (INFO default). Eine Konfiguration @@ -68,6 +68,7 @@ app.include_router(system.router) app.include_router(connect.router) app.include_router(memory.router) app.include_router(agent.router) +app.include_router(voice.router) # Sprache: STT/TTS-Proxy + Hermes-Agent-Chat (Voice-Tab) app.include_router(gateway_proxy.router) # OpenAI-kompatibler /v1-Gateway (model:auto) app.include_router(maintenance.router) diff --git a/backend/config.py b/backend/config.py index ef897a4..1babc44 100644 --- a/backend/config.py +++ b/backend/config.py @@ -60,6 +60,35 @@ GATEWAY_URL = os.environ.get("MC_GATEWAY_URL", f"http://127.0.0.1:{os.environ.ge # --- Hermes Agent (eigener Dienst auf der Box) ------------------------------- # Gateway (OpenAI-API des Agenten) + interaktives Web-Terminal (ttyd → `hermes chat`). HERMES_API_URL = os.environ.get("HERMES_API_URL", "http://127.0.0.1:8642").rstrip("/") +# API-Key der Hermes-`api_server`-Plattform (~/.hermes/.env: API_SERVER_KEY). Nötig für +# /v1/chat/completions (Voice-Pipeline) — Bearer-Auth, sonst 401. Derselbe volle Agent +# (Tools + geteiltes Mem0) wie CLI/Telegram, nur über HTTP. +def _read_hermes_env(key: str) -> str: + """Liest einen Schlüssel aus ~/.hermes/.env (Fallback, falls nicht in der Prozess-Env). + Der MC2-Dienst erbt die Hermes-Secrets sonst nicht.""" + try: + env_path = Path(os.path.expanduser(os.environ.get("HERMES_HOME", "~/.hermes"))) / ".env" + for line in env_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith(f"{key}="): + return line.split("=", 1)[1].strip().strip('"').strip("'") + except OSError: + pass + return "" + + +HERMES_API_KEY = ( + os.environ.get("HERMES_API_KEY") + or os.environ.get("API_SERVER_KEY") + or _read_hermes_env("API_SERVER_KEY") +) +# Modellfeld im OpenAI-Request; die api_server-Plattform nutzt ihr konfiguriertes Hirn, +# das Feld ist i.d.R. kosmetisch. Override via Env, falls die Plattform strikt prüft. +HERMES_API_MODEL = os.environ.get("HERMES_API_MODEL", "hermes") + +# --- Voice-Sidecar (STT faster-whisper + TTS Piper/Chatterbox) --------------- +# Eigenes Python-3.12-venv (~/.voice/venv), analog Mem0-Sidecar. MC2 proxyt nach außen. +VOICE_SERVICE_URL = os.environ.get("MC_VOICE_SERVICE_URL", "http://127.0.0.1:8650").rstrip("/") # Hermes-Terminal: ttyd-Web-Terminal der interaktiven Agent-CLI (Ersatz für AnythingLLM-Chat). # Wird in MC2 per iframe eingebettet (Terminal-Seite). Siehe deploy/hermes-terminal.service. HERMES_TERMINAL_URL = os.environ.get("MC_HERMES_TERMINAL_URL", "http://192.168.178.151:7681").rstrip("/") diff --git a/backend/routers/system.py b/backend/routers/system.py index 167ed2f..dc3368a 100644 --- a/backend/routers/system.py +++ b/backend/routers/system.py @@ -14,7 +14,7 @@ from pydantic import BaseModel import httpx -from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL +from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL, VOICE_SERVICE_URL from services import backup as backup_svc from services.agent import agent_status from services.gateway import gateway_reachable @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) router = APIRouter(prefix="/api") # Nur diese User-Dienste dürfen neugestartet werden. -ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui"} +ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui", "mem0-service", "voice-service"} # Quelle für Self-Update (auf der Box ~/mission-control-v2). SOURCE_DIR = os.path.expanduser(os.environ.get("MC2_SOURCE_DIR", "~/mission-control-v2")) @@ -45,6 +45,13 @@ def _mem0_reachable() -> bool: return False +def _voice_reachable() -> bool: + try: + return httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=2).status_code == 200 + except Exception: + return False + + @router.get("/system/services") def services() -> dict: """Aggregierte Erreichbarkeit aller Stack-Dienste (für die Health-Anzeige).""" @@ -57,6 +64,7 @@ def services() -> dict: {"name": "Hermes-Gateway", "url": HERMES_API_URL, "ok": a["gateway_reachable"]}, {"name": "Hermes-Terminal", "url": a["terminal_url"], "ok": a["terminal_reachable"]}, {"name": "Mem0 (Gedächtnis)", "url": MEM0_SERVICE_URL, "ok": _mem0_reachable()}, + {"name": "Voice (STT/TTS)", "url": VOICE_SERVICE_URL, "ok": _voice_reachable()}, ], "links": { "engine_ui": f"{LLAMA_SWAP_URL}/ui", diff --git a/backend/routers/voice.py b/backend/routers/voice.py new file mode 100644 index 0000000..c9b13e9 --- /dev/null +++ b/backend/routers/voice.py @@ -0,0 +1,130 @@ +""" +Voice-Endpoints für „Mit Hermes reden" (Browser-Voice + 3D-Avatar). + +Dünner Layer: STT/TTS werden zum Voice-Sidecar (:8650) geproxyt; der Chat geht an den +Hermes-`api_server` (:8642, OpenAI-kompatibel) — denselben vollen Agenten mit Tools + +geteiltem Mem0 wie CLI/Telegram. Mit stabilem `X-Hermes-Session-Id` hält die Plattform den +Transcript server-seitig, daher schickt der Client je Turn nur die neue User-Nachricht. + +LAN-only (kein Token in der 2.0-Phase), wie die übrigen MC2-Endpoints. +""" + +import logging + +import httpx +from fastapi import APIRouter, File, Form, HTTPException, UploadFile +from fastapi.responses import Response, StreamingResponse +from pydantic import BaseModel + +from config import HERMES_API_KEY, HERMES_API_MODEL, HERMES_API_URL, VOICE_SERVICE_URL + +log = logging.getLogger(__name__) +router = APIRouter(prefix="/api") + +_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # Chatterbox-TTS auf CPU darf dauern + + +class TTSIn(BaseModel): + text: str + engine: str = "piper" + voice: str = "" + language: str = "" + ref_path: str = "" + + +class ChatIn(BaseModel): + text: str # die neue User-Äußerung (STT-Ergebnis) + session_id: str # stabiler Voice-Faden → server-seitiger Transcript + session_key: str = "" # optional: Langzeit-Memory-Scope + system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen") + model: str = "" + + +@router.get("/voice/health") +def voice_health() -> dict: + """Erreichbarkeit des Voice-Sidecars + ob der Hermes-API-Key gesetzt ist.""" + out: dict = {"sidecar": False, "hermes_key": bool(HERMES_API_KEY)} + try: + r = httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=httpx.Timeout(5.0)) + out["sidecar"] = r.status_code == 200 + out["detail"] = r.json() if r.status_code == 200 else None + except Exception as exc: # noqa: BLE001 + out["error"] = str(exc) + return out + + +@router.get("/voice/voices") +def voice_voices() -> dict: + try: + r = httpx.get(f"{VOICE_SERVICE_URL}/voices", timeout=httpx.Timeout(10.0)) + r.raise_for_status() + return r.json() + except Exception as exc: # noqa: BLE001 + raise HTTPException(502, f"Voice-Sidecar nicht erreichbar: {exc}") + + +@router.post("/voice/stt") +async def voice_stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict: + """Mikro-Audio → Text (Proxy auf Sidecar /stt).""" + data = await audio.read() + if not data: + raise HTTPException(400, "Leeres Audio.") + files = {"audio": (audio.filename or "rec.webm", data, audio.content_type or "audio/webm")} + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + r = await client.post(f"{VOICE_SERVICE_URL}/stt", files=files, data={"language": language}) + r.raise_for_status() + return r.json() + except httpx.HTTPError as exc: + raise HTTPException(502, f"STT fehlgeschlagen: {exc}") + + +@router.post("/voice/tts") +async def voice_tts(body: TTSIn) -> Response: + """Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes.""" + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + r = await client.post(f"{VOICE_SERVICE_URL}/tts", json=body.model_dump()) + r.raise_for_status() + return Response(content=r.content, media_type=r.headers.get("content-type", "audio/wav")) + except httpx.HTTPError as exc: + raise HTTPException(502, f"TTS fehlgeschlagen: {exc}") + + +@router.post("/voice/chat") +async def voice_chat(body: ChatIn) -> StreamingResponse: + """Neue User-Äußerung → Hermes-Agent (api_server, streamend). SSE wird 1:1 durchgereicht. + + Mit `X-Hermes-Session-Id` hält die Plattform den Verlauf — wir senden nur die neue Nachricht. + Auth per Bearer (API_SERVER_KEY); ohne Key liefert :8642 ein 401.""" + if not HERMES_API_KEY: + raise HTTPException(503, "HERMES_API_KEY/API_SERVER_KEY nicht gesetzt — Agent-Auth fehlt.") + + messages = [] + if body.system: + messages.append({"role": "system", "content": body.system}) + messages.append({"role": "user", "content": body.text}) + payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True} + headers = { + "Authorization": f"Bearer {HERMES_API_KEY}", + "X-Hermes-Session-Id": body.session_id, + } + if body.session_key: + headers["X-Hermes-Session-Key"] = body.session_key + + async def gen(): + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(None, connect=5.0)) as client: + async with client.stream( + "POST", f"{HERMES_API_URL}/v1/chat/completions", json=payload, headers=headers, + ) as r: + if r.status_code != 200: + detail = (await r.aread()).decode("utf-8", "replace")[:500] + yield f"data: {{\"error\": \"Hermes {r.status_code}: {detail}\"}}\n\n".encode() + return + async for chunk in r.aiter_raw(): + yield chunk + except httpx.HTTPError as exc: + yield f"data: {{\"error\": \"Verbindung zu Hermes fehlgeschlagen: {exc}\"}}\n\n".encode() + + return StreamingResponse(gen(), media_type="text/event-stream") diff --git a/backend/services/maintenance.py b/backend/services/maintenance.py index 1dd358a..4dd2b33 100644 --- a/backend/services/maintenance.py +++ b/backend/services/maintenance.py @@ -19,7 +19,7 @@ from services import catalog, discover, jobengine, llamaswap, system # System-Dienste (root, via sudo -n NOPASSWD) vs. User-Dienste (systemctl --user). SYSTEM_SERVICES = {"llama-swap"} -USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service"} +USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service", "voice-service"} # Engine-Update: lädt den neuesten Vulkan-Build (deploy/update-engine.sh, läuft als root). _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) diff --git a/deploy/deploy.sh b/deploy/deploy.sh index ca6a2dd..1bcb8e5 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -31,6 +31,12 @@ else echo "WARN: ~/.mem0/venv fehlt — Mem0-Sidecar wird nicht gestartet (siehe Plan A1)." fi +# Voice-Sidecar (lokales STT + gestuftes TTS für „Mit Hermes reden") — eigenes Python-3.12-venv +# (~/.voice/venv), weil torch/chatterbox/faster-whisper nicht ins 3.14-Backend-venv passen. +# install.sh ist idempotent (venv + Deps + Piper-Stimmen). Best-effort: schlägt es fehl, läuft +# der restliche Stack weiter (der Voice-Tab meldet den Sidecar dann als offline). +bash "$SRC/voice_service/install.sh" || echo "WARN: Voice-Sidecar-Setup fehlgeschlagen — Voice-Tab bleibt offline." + # Hermes-Memory-Provider-Plugin (auto-lernen/Recall via Mem0-Sidecar) nach ~/.hermes/plugins/ # spiegeln. Context-only (kein Tool-Loop). Aktivierung in ~/.hermes/config.yaml: # memory.memory_enabled: true + memory.provider: mc2-memory (einmalig, box-lokal). @@ -48,6 +54,8 @@ cp "$SRC/deploy/mission-control-2.service" "$HOME/.config/systemd/user/mission-c cp "$SRC/deploy/hermes-terminal.service" "$HOME/.config/systemd/user/hermes-terminal.service" # Mem0-Sidecar-Unit (nur wenn das venv existiert). [ -x "$HOME/.mem0/venv/bin/python" ] && cp "$SRC/deploy/mem0-service.service" "$HOME/.config/systemd/user/mem0-service.service" +# Voice-Sidecar-Unit (nur wenn das venv existiert). +[ -x "$HOME/.voice/venv/bin/python" ] && cp "$SRC/deploy/voice-service.service" "$HOME/.config/systemd/user/voice-service.service" # Tägliches Zustands-Backup (mem0 + Configs/Secrets) — Timer + oneshot-Service. Siehe docs/BACKUP.md. cp "$SRC/deploy/mc2-backup.service" "$HOME/.config/systemd/user/mc2-backup.service" cp "$SRC/deploy/mc2-backup.timer" "$HOME/.config/systemd/user/mc2-backup.timer" @@ -55,10 +63,13 @@ systemctl --user daemon-reload systemctl --user enable mission-control-2 >/dev/null 2>&1 || true systemctl --user enable hermes-terminal >/dev/null 2>&1 || true systemctl --user enable mem0-service >/dev/null 2>&1 || true +systemctl --user enable voice-service >/dev/null 2>&1 || true systemctl --user enable --now mc2-backup.timer >/dev/null 2>&1 || true loginctl enable-linger "$USER" >/dev/null 2>&1 || true # Mem0-Sidecar VOR dem Backend (re)starten, damit /api/memory sofort bedient wird. [ -x "$HOME/.mem0/venv/bin/python" ] && systemctl --user restart mem0-service 2>/dev/null || true +# Voice-Sidecar (re)starten (best-effort; Erststart lädt das STT-Modell vor). +[ -x "$HOME/.voice/venv/bin/python" ] && systemctl --user restart voice-service 2>/dev/null || true systemctl --user restart mission-control-2 command -v ttyd >/dev/null 2>&1 && systemctl --user restart hermes-terminal 2>/dev/null || true diff --git a/deploy/voice-service.service b/deploy/voice-service.service new file mode 100644 index 0000000..180883c --- /dev/null +++ b/deploy/voice-service.service @@ -0,0 +1,23 @@ +[Unit] +Description=MC2 Voice Sidecar — lokales STT (faster-whisper) + gestuftes TTS (Piper/Chatterbox) +After=network.target + +[Service] +# Eigenes Python-3.12-venv (~/.voice/venv) — torch/chatterbox/faster-whisper passen nicht ins +# 3.14-Backend-venv (analog mem0-service). Bind 127.0.0.1: nur lokal; MC2 proxyt nach außen. +Type=simple +WorkingDirectory=%h/mission-control-v2/voice_service +Environment=VOICE_PORT=8650 +Environment=VOICE_STT_MODEL=medium +Environment=VOICE_STT_LANG=de +Environment=VOICE_PIPER_DIR=%h/.voice/voices +Environment=VOICE_PIPER_DEFAULT=de_DE-thorsten-medium +Environment=VOICE_CHATTERBOX_DEVICE=cpu +Environment=VOICE_CHATTERBOX_LANG=de +Environment=TOKENIZERS_PARALLELISM=false +ExecStart=%h/.voice/venv/bin/python -m uvicorn app:app --host 127.0.0.1 --port 8650 +Restart=always +RestartSec=3 + +[Install] +WantedBy=default.target diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9d2deb2..0b572ee 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,9 @@ "name": "mission-control-2-frontend", "version": "2.0.0", "dependencies": { + "@pixiv/three-vrm": "^3.4.0", + "@react-three/drei": "^9.114.0", + "@react-three/fiber": "^8.17.10", "@tanstack/react-query": "^5.101.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -16,13 +19,15 @@ "react-dom": "^18.3.1", "reagraph": "^4.22.0", "recharts": "^3.9.0", - "tailwind-merge": "^2.5.5" + "tailwind-merge": "^2.5.5", + "three": "^0.169.0" }, "devDependencies": { "@tailwindcss/vite": "^4.0.0", "@types/node": "^22.10.1", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", + "@types/three": "^0.169.0", "@vitejs/plugin-react": "^4.3.4", "tailwindcss": "^4.0.0", "typescript": "^5.6.3", @@ -320,12 +325,6 @@ "node": ">=6.9.0" } }, - "node_modules/@dimforge/rapier3d-compat": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz", - "integrity": "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==", - "license": "Apache-2.0" - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", @@ -824,6 +823,154 @@ "integrity": "sha512-Rp7ll8BHrKB3wXaRFKhrltwZl1CiXGdibPxuWXvqGnKTnv8fqa/nvftYNuSbf+pbJWKYCXdBtYTITdAUTGGh0Q==", "license": "Apache-2.0" }, + "node_modules/@monogrid/gainmap-js": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@monogrid/gainmap-js/-/gainmap-js-3.4.0.tgz", + "integrity": "sha512-2Z0FATFHaoYJ8b+Y4y4Hgfn3FRFwuU5zRrk+9dFWp4uGAdHGqVEdP7HP+gLA3X469KXHmfupJaUbKo1b/aDKIg==", + "license": "MIT", + "dependencies": { + "promise-worker-transferable": "^1.0.4" + }, + "peerDependencies": { + "three": ">= 0.159.0" + } + }, + "node_modules/@pixiv/three-vrm": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm/-/three-vrm-3.5.4.tgz", + "integrity": "sha512-hY0MnmKVLUebFR9QN9vCRXo0rZmLpxjLVZAupQv5qDDGCKZuNxlOts9PqTkWlzhkMLLdhB48AF+P4PCy/DFOzA==", + "license": "MIT", + "dependencies": { + "@pixiv/three-vrm-core": "3.5.4", + "@pixiv/three-vrm-materials-hdr-emissive-multiplier": "3.5.4", + "@pixiv/three-vrm-materials-mtoon": "3.5.4", + "@pixiv/three-vrm-materials-v0compat": "3.5.4", + "@pixiv/three-vrm-node-constraint": "3.5.4", + "@pixiv/three-vrm-springbone": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-core": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-core/-/three-vrm-core-3.5.4.tgz", + "integrity": "sha512-CgaxZ4qX6JmE2oKsOVGGlheT011qP4bBZGUaMJJ5iQINI999+cveIdISe459B+AMYD1dU5h1+xCtDthLQNH1Bg==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrm-0.0": "3.5.4", + "@pixiv/types-vrmc-vrm-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-materials-hdr-emissive-multiplier": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-hdr-emissive-multiplier/-/three-vrm-materials-hdr-emissive-multiplier-3.5.4.tgz", + "integrity": "sha512-3BPZ42qW38cHhP+imqEnqTFsltvYkHA/t4VzWFdQ9sngWt0NiFAOotEsv8gZn1ZgKE9VRQezXne6s5aRWQlQqA==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-materials-mtoon": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-mtoon/-/three-vrm-materials-mtoon-3.5.4.tgz", + "integrity": "sha512-vLHt7IYZxlijbCMa5TRRf6gaQjA65F/d7oZBNfZW3XC8sBg57ZV0M/xBAJm2d6UwVmTbBd9NG9li/Y9yuupDQw==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrm-0.0": "3.5.4", + "@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-materials-v0compat": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-v0compat/-/three-vrm-materials-v0compat-3.5.4.tgz", + "integrity": "sha512-qRRtg8vYFBRJpsa3evruWqmXr0Gwd8uJXwxxUKuKtrIN0Hw0hJsRLHkg9E238Q8xRTnoycdkSgX4VvVknvtXBA==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrm-0.0": "3.5.4", + "@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-node-constraint": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-node-constraint/-/three-vrm-node-constraint-3.5.4.tgz", + "integrity": "sha512-nyAghDrYNp0Z2siEaY2+th+FzZdDs9EJy8K576Mpw9cy3jwtCJuTHBQmGzKkrgoLaBfrso+3Tdfgr62AJV0m1w==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrmc-node-constraint-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/three-vrm-springbone": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/three-vrm-springbone/-/three-vrm-springbone-3.5.4.tgz", + "integrity": "sha512-8KWA7vHU+OnW6XcZPnDuUrjjPFRzmv2yxcTMNCYQvBS547/Iqo0RLbQvpO/ppceLTNJFY2TGXAWJU67aeazWjw==", + "license": "MIT", + "dependencies": { + "@pixiv/types-vrm-0.0": "3.5.4", + "@pixiv/types-vrmc-springbone-1.0": "3.5.4", + "@pixiv/types-vrmc-springbone-extended-collider-1.0": "3.5.4" + }, + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/@pixiv/types-vrm-0.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrm-0.0/-/types-vrm-0.0-3.5.4.tgz", + "integrity": "sha512-g9VMPikJxKJ/XgnhXxnqj6ejVhQ9WwOEUp7KyyZHsqRnHcQfSqyZFTs9wkAMCbwCqHXX/lg9+9Ogj8KhxCkbXw==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0/-/types-vrmc-materials-hdr-emissive-multiplier-1.0-3.5.4.tgz", + "integrity": "sha512-h9GEQ3q1VTylL/P40kJ8uoQhfUYY54NhTG6Xsnl4X0jf7oHh8MKXYnMMn903hXml6082Yjx1C6x3DKckdnophg==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-materials-mtoon-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-materials-mtoon-1.0/-/types-vrmc-materials-mtoon-1.0-3.5.4.tgz", + "integrity": "sha512-AU5sOcsbmcnzRPfgfk7fzKhk5lNm2MqrbvtAsSlhylVMdlzrmXqIfRuexABcB9k+ysTp49qIFioot9KRlQcDUw==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-node-constraint-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-node-constraint-1.0/-/types-vrmc-node-constraint-1.0-3.5.4.tgz", + "integrity": "sha512-RRbK5NNvZv4ewRELezueCiDB11FGkt4pdZR/UJ027DJPaNzd6rls2OEJ8weQ3OmgHZpbx/BtdpX2JWX5WZZd/Q==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-springbone-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-springbone-1.0/-/types-vrmc-springbone-1.0-3.5.4.tgz", + "integrity": "sha512-NO7HTRBuWEe89Wo9BRI5hX1kWVkZzA4YWg9XALSTLdIT8HMiArL4NRse/CF91jNuLcTGfPEWlBJyq9SciAQxAg==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-springbone-extended-collider-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-springbone-extended-collider-1.0/-/types-vrmc-springbone-extended-collider-1.0-3.5.4.tgz", + "integrity": "sha512-uzJmcRh/iHYnZPtnq6N1+jb+T/ptYNCyFEoArkWGNXHSfPjTkMcheTQkL5iMNvTE6hqhyMRVMha9OmbvyLmmUQ==", + "license": "MIT" + }, + "node_modules/@pixiv/types-vrmc-vrm-1.0": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-vrm-1.0/-/types-vrmc-vrm-1.0-3.5.4.tgz", + "integrity": "sha512-pkjT4QXT/Hp6rcq8J8EFEHIldbtPGYWOTaWdoRGMA8/KzJI5PdekZ6AUYbnSP6ElfJ3BSZ01DxxpQZa/UNsFqA==", + "license": "MIT" + }, "node_modules/@radix-ui/primitive": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.4.tgz", @@ -1205,12 +1352,236 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@react-spring/three": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.5.tgz", + "integrity": "sha512-RxIsCoQfUqOS3POmhVHa1wdWS0wyHAUway73uRLp3GAL5U2iYVNdnzQsep6M2NZ994BlW8TcKuMtQHUqOsy6WA==", + "license": "MIT", + "dependencies": { + "@react-spring/animated": "~9.7.5", + "@react-spring/core": "~9.7.5", + "@react-spring/shared": "~9.7.5", + "@react-spring/types": "~9.7.5" + }, + "peerDependencies": { + "@react-three/fiber": ">=6.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "three": ">=0.126" + } + }, + "node_modules/@react-spring/three/node_modules/@react-spring/animated": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.5.tgz", + "integrity": "sha512-Tqrwz7pIlsSDITzxoLS3n/v/YCUHQdOIKtOJf4yL6kYVSDTSmVK1LI1Q3M/uu2Sx4X3pIWF3xLUhlsA6SPNTNg==", + "license": "MIT", + "dependencies": { + "@react-spring/shared": "~9.7.5", + "@react-spring/types": "~9.7.5" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/@react-spring/three/node_modules/@react-spring/core": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.5.tgz", + "integrity": "sha512-rmEqcxRcu7dWh7MnCcMXLvrf6/SDlSokLaLTxiPlAYi11nN3B5oiCUAblO72o+9z/87j2uzxa2Inm8UbLjXA+w==", + "license": "MIT", + "dependencies": { + "@react-spring/animated": "~9.7.5", + "@react-spring/shared": "~9.7.5", + "@react-spring/types": "~9.7.5" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/react-spring/donate" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/@react-spring/three/node_modules/@react-spring/rafz": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.5.tgz", + "integrity": "sha512-5ZenDQMC48wjUzPAm1EtwQ5Ot3bLIAwwqP2w2owG5KoNdNHpEJV263nGhCeKKmuA3vG2zLLOdu3or6kuDjA6Aw==", + "license": "MIT" + }, + "node_modules/@react-spring/three/node_modules/@react-spring/shared": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.5.tgz", + "integrity": "sha512-wdtoJrhUeeyD/PP/zo+np2s1Z820Ohr/BbuVYv+3dVLW7WctoiN7std8rISoYoHpUXtbkpesSKuPIw/6U1w1Pw==", + "license": "MIT", + "dependencies": { + "@react-spring/rafz": "~9.7.5", + "@react-spring/types": "~9.7.5" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/@react-spring/three/node_modules/@react-spring/types": { + "version": "9.7.5", + "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.5.tgz", + "integrity": "sha512-HVj7LrZ4ReHWBimBvu2SKND3cDVUPWKLqRTmWe/fNY6o1owGOX0cAHbdPDTMelgBlVbrTKrre6lFkhqGZErK/g==", + "license": "MIT" + }, "node_modules/@react-spring/types": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.6.1.tgz", "integrity": "sha512-POu8Mk0hIU3lRXB3bGIGe4VHIwwDsQyoD1F394OK7STTiX9w4dG3cTLljjYswkQN+hDSHRrj4O36kuVa7KPU8Q==", "license": "MIT" }, + "node_modules/@react-three/drei": { + "version": "9.122.0", + "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.122.0.tgz", + "integrity": "sha512-SEO/F/rBCTjlLez7WAlpys+iGe9hty4rNgjZvgkQeXFSiwqD4Hbk/wNHMAbdd8vprO2Aj81mihv4dF5bC7D0CA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mediapipe/tasks-vision": "0.10.17", + "@monogrid/gainmap-js": "^3.0.6", + "@react-spring/three": "~9.7.5", + "@use-gesture/react": "^10.3.1", + "camera-controls": "^2.9.0", + "cross-env": "^7.0.3", + "detect-gpu": "^5.0.56", + "glsl-noise": "^0.0.0", + "hls.js": "^1.5.17", + "maath": "^0.10.8", + "meshline": "^3.3.1", + "react-composer": "^5.0.3", + "stats-gl": "^2.2.8", + "stats.js": "^0.17.0", + "suspend-react": "^0.1.3", + "three-mesh-bvh": "^0.7.8", + "three-stdlib": "^2.35.6", + "troika-three-text": "^0.52.0", + "tunnel-rat": "^0.1.2", + "utility-types": "^3.11.0", + "zustand": "^5.0.1" + }, + "peerDependencies": { + "@react-three/fiber": "^8", + "react": "^18", + "react-dom": "^18", + "three": ">=0.137" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": { + "version": "0.10.17", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz", + "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==", + "license": "Apache-2.0" + }, + "node_modules/@react-three/drei/node_modules/zustand": { + "version": "5.0.14", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.14.tgz", + "integrity": "sha512-/8tAspM5LMPr28b3fwLYrtdj77ECpfZviaP75CMTnwO8ISyaE4GDIG/9rDDYq/cH9D2Xw2A2RXglLInmVBQB/g==", + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + }, + "node_modules/@react-three/fiber": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-8.18.0.tgz", + "integrity": "sha512-FYZZqD0UUHUswKz3LQl2Z7H24AhD14XGTsIRw3SJaXUxyfVMi+1yiZGmqTcPt/CkPpdU7rrxqcyQ1zJE5DjvIQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.17.8", + "@types/react-reconciler": "^0.26.7", + "@types/webxr": "*", + "base64-js": "^1.5.1", + "buffer": "^6.0.3", + "its-fine": "^1.0.6", + "react-reconciler": "^0.27.0", + "react-use-measure": "^2.1.7", + "scheduler": "^0.21.0", + "suspend-react": "^0.1.3", + "zustand": "^3.7.1" + }, + "peerDependencies": { + "expo": ">=43.0", + "expo-asset": ">=8.4", + "expo-file-system": ">=11.0", + "expo-gl": ">=11.0", + "react": ">=18 <19", + "react-dom": ">=18 <19", + "react-native": ">=0.64", + "three": ">=0.133" + }, + "peerDependenciesMeta": { + "expo": { + "optional": true + }, + "expo-asset": { + "optional": true + }, + "expo-file-system": { + "optional": true + }, + "expo-gl": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "react-native": { + "optional": true + } + } + }, + "node_modules/@react-three/fiber/node_modules/scheduler": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.21.0.tgz", + "integrity": "sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/@react-three/fiber/node_modules/zustand": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz", + "integrity": "sha512-PIJDIZKtokhof+9+60cpockVOq05sJzHCriyvaLBmEJixseQ1a5Kdov6fWZfWOu5SK9c+FhH1jU0tntLxRJYMA==", + "license": "MIT", + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/@reduxjs/toolkit": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", @@ -2150,17 +2521,17 @@ "license": "MIT" }, "node_modules/@types/three": { - "version": "0.185.0", - "resolved": "https://registry.npmjs.org/@types/three/-/three-0.185.0.tgz", - "integrity": "sha512-O2Uy8Cj4Nonr8dWUUbifMdPe8B0Mq7EdOHb89S4+kjUw/KhbjTZrUuYlrQ1bpUKG+EP9QJnN7qNxbHGlGoLHMA==", + "version": "0.169.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.169.0.tgz", + "integrity": "sha512-oan7qCgJBt03wIaK+4xPWclYRPG9wzcg7Z2f5T8xYTNEF95kh0t0lklxLLYBDo7gQiGLYzE6iF4ta7nXF2bcsw==", "license": "MIT", "dependencies": { - "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", - "@types/webxr": ">=0.5.17", + "@types/webxr": "*", + "@webgpu/types": "*", "fflate": "~0.8.2", - "meshoptimizer": "~1.1.1" + "meshoptimizer": "~0.18.1" } }, "node_modules/@types/three/node_modules/fflate": { @@ -2220,6 +2591,12 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.71", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz", + "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==", + "license": "BSD-3-Clause" + }, "node_modules/@yomguithereal/helpers": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@yomguithereal/helpers/-/helpers-1.1.1.tgz", @@ -2238,6 +2615,26 @@ "node": ">=10" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/baseline-browser-mapping": { "version": "2.10.38", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.38.tgz", @@ -2294,6 +2691,30 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/camera-controls": { "version": "2.10.1", "resolved": "https://registry.npmjs.org/camera-controls/-/camera-controls-2.10.1.tgz", @@ -2916,12 +3337,44 @@ "graphology-types": ">=0.23.0" } }, + "node_modules/hls.js": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz", + "integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==", + "license": "Apache-2.0" + }, "node_modules/hold-event": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/hold-event/-/hold-event-0.2.0.tgz", "integrity": "sha512-rko5P1XgHzy4B0NR0xVHEpWPgj0i23f8Mf8qsOugd1CHvfLR0PyIyy+8TAQQA9v8qAa1OZ4XuCKk04rxmPGHNQ==", "license": "MIT" }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/immer": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz", @@ -2941,6 +3394,12 @@ "node": ">=12" } }, + "node_modules/is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", + "license": "MIT" + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3010,6 +3469,15 @@ "node": ">=6" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -3314,6 +3782,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc" } }, + "node_modules/maath": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/maath/-/maath-0.10.8.tgz", + "integrity": "sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g==", + "license": "MIT", + "peerDependencies": { + "@types/three": ">=0.134.0", + "three": ">=0.134.0" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -3324,10 +3802,19 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/meshline": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz", + "integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.137" + } + }, "node_modules/meshoptimizer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.1.1.tgz", - "integrity": "sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-0.18.1.tgz", + "integrity": "sha512-ZhoIoL7TNV4s5B6+rx5mC//fw8/POGyNxS/DZyCJeiZ12ScLfVwRE/GfsxwiTkMYYD5DmK2/JXnEVXqL4rF+Sw==", "license": "MIT" }, "node_modules/mnemonist": { @@ -3463,6 +3950,16 @@ "integrity": "sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==", "license": "ISC" }, + "node_modules/promise-worker-transferable": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/promise-worker-transferable/-/promise-worker-transferable-1.0.4.tgz", + "integrity": "sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw==", + "license": "Apache-2.0", + "dependencies": { + "is-promise": "^2.1.0", + "lie": "^3.0.2" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -3826,34 +4323,6 @@ } } }, - "node_modules/reagraph/node_modules/glodrei/node_modules/maath": { - "version": "0.10.8", - "resolved": "https://registry.npmjs.org/maath/-/maath-0.10.8.tgz", - "integrity": "sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g==", - "license": "MIT", - "peerDependencies": { - "@types/three": ">=0.134.0", - "three": ">=0.134.0" - } - }, - "node_modules/reagraph/node_modules/glodrei/node_modules/meshline": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz", - "integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==", - "license": "MIT", - "peerDependencies": { - "three": ">=0.137" - } - }, - "node_modules/reagraph/node_modules/glodrei/node_modules/three-mesh-bvh": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/three-mesh-bvh/-/three-mesh-bvh-0.7.6.tgz", - "integrity": "sha512-rCjsnxEqR9r1/C/lCqzGLS67NDty/S/eT6rAJfDvsanrIctTWdNoR4ZOGWewCB13h1QkVo2BpmC0wakj1+0m8A==", - "license": "MIT", - "peerDependencies": { - "three": ">= 0.151.0" - } - }, "node_modules/reagraph/node_modules/glodrei/node_modules/troika-three-text": { "version": "0.47.2", "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.47.2.tgz", @@ -3904,6 +4373,12 @@ "loose-envify": "^1.1.0" } }, + "node_modules/reagraph/node_modules/three": { + "version": "0.154.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz", + "integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==", + "license": "MIT" + }, "node_modules/recharts": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.9.0.tgz", @@ -4126,11 +4601,21 @@ } }, "node_modules/three": { - "version": "0.154.0", - "resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz", - "integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==", + "version": "0.169.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.169.0.tgz", + "integrity": "sha512-Ed906MA3dR4TS5riErd4QBsRGPcx+HBDX2O5yYE5GqJeFQTPU+M56Va/f/Oph9X7uZo3W3o4l2ZhBZ6f6qUv0w==", "license": "MIT" }, + "node_modules/three-mesh-bvh": { + "version": "0.7.8", + "resolved": "https://registry.npmjs.org/three-mesh-bvh/-/three-mesh-bvh-0.7.8.tgz", + "integrity": "sha512-BGEZTOIC14U0XIRw3tO4jY7IjP7n7v24nv9JXS1CyeVRWOCkcOMhRnmENUjuV39gktAw4Ofhr0OvIAiTspQrrw==", + "deprecated": "Deprecated due to three.js version incompatibility. Please use v0.8.0, instead.", + "license": "MIT", + "peerDependencies": { + "three": ">= 0.151.0" + } + }, "node_modules/three-stdlib": { "version": "2.36.1", "resolved": "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.36.1.tgz", @@ -4171,6 +4656,36 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/troika-three-text": { + "version": "0.52.4", + "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.52.4.tgz", + "integrity": "sha512-V50EwcYGruV5rUZ9F4aNsrytGdKcXKALjEtQXIOBfhVoZU9VAqZNIoGQ3TMiooVqFAbR1w15T+f+8gkzoFzawg==", + "license": "MIT", + "dependencies": { + "bidi-js": "^1.0.2", + "troika-three-utils": "^0.52.4", + "troika-worker-utils": "^0.52.0", + "webgl-sdf-generator": "1.1.1" + }, + "peerDependencies": { + "three": ">=0.125.0" + } + }, + "node_modules/troika-three-text/node_modules/troika-worker-utils": { + "version": "0.52.0", + "resolved": "https://registry.npmjs.org/troika-worker-utils/-/troika-worker-utils-0.52.0.tgz", + "integrity": "sha512-W1CpvTHykaPH5brv5VHLfQo9D1OYuo0cSBEUQFFT/nBUzM8iD6Lq2/tgG/f1OelbAS1WtaTPQzE5uM49egnngw==", + "license": "MIT" + }, + "node_modules/troika-three-utils": { + "version": "0.52.4", + "resolved": "https://registry.npmjs.org/troika-three-utils/-/troika-three-utils-0.52.4.tgz", + "integrity": "sha512-NORAStSVa/BDiG52Mfudk4j1FG4jC4ILutB3foPnfGbOeIs9+G5vZLa0pnmnaftZUGm4UwSoqEpWdqvC7zms3A==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.125.0" + } + }, "node_modules/troika-worker-utils": { "version": "0.47.2", "resolved": "https://registry.npmjs.org/troika-worker-utils/-/troika-worker-utils-0.47.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 1be38ad..5dc76dd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,9 @@ "preview": "vite preview" }, "dependencies": { + "@pixiv/three-vrm": "^3.4.0", + "@react-three/drei": "^9.114.0", + "@react-three/fiber": "^8.17.10", "@tanstack/react-query": "^5.101.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -17,13 +20,15 @@ "react-dom": "^18.3.1", "reagraph": "^4.22.0", "recharts": "^3.9.0", - "tailwind-merge": "^2.5.5" + "tailwind-merge": "^2.5.5", + "three": "^0.169.0" }, "devDependencies": { "@tailwindcss/vite": "^4.0.0", "@types/node": "^22.10.1", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", + "@types/three": "^0.169.0", "@vitejs/plugin-react": "^4.3.4", "tailwindcss": "^4.0.0", "typescript": "^5.6.3", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3a50173..2c690d4 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -8,6 +8,7 @@ import { ConnectView } from "@/views/ConnectView" import { MemoryView } from "@/views/MemoryView" import { AgentView } from "@/views/AgentView" import { TerminalView } from "@/views/TerminalView" +import { VoiceView } from "@/views/VoiceView" import { GuideView } from "@/views/GuideView" import { Placeholder } from "@/views/Placeholder" import { SystemDrawer } from "@/components/SystemDrawer" @@ -191,8 +192,9 @@ export default function App() { {view === "memory" && } {view === "agent" && } {view === "terminal" && } + {view === "voice" && } {view === "guide" && } - {!["dashboard", "models", "connect", "memory", "agent", "terminal", "guide"].includes(view) && ( + {!["dashboard", "models", "connect", "memory", "agent", "terminal", "voice", "guide"].includes(view) && ( )} diff --git a/frontend/src/components/voice/Avatar3D.tsx b/frontend/src/components/voice/Avatar3D.tsx new file mode 100644 index 0000000..ba9857f --- /dev/null +++ b/frontend/src/components/voice/Avatar3D.tsx @@ -0,0 +1,139 @@ +import { Canvas, useFrame } from "@react-three/fiber" +import { OrbitControls } from "@react-three/drei" +import { useEffect, useRef, useState, type MutableRefObject } from "react" +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js" +import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" +import type { Emotion } from "@/lib/voice/sentiment" + +// 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und +// stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame). + +type LevelRef = MutableRefObject<{ current: number }> // audioLevel.current.current = Pegel 0..1 +type EmotionRef = MutableRefObject + +// VRMs laden in T-Pose (Bindepose, Arme waagerecht). Wir senken Ober-/Unterarme auf den +// normalisierten Humanoid-Knoten zu einer ruhigen A-Pose ab — sieht sofort natürlich aus. +// (Echte Idle-Animation wäre die spätere Stufe.) +function applyRestPose(vrm: VRM) { + const set = (name: any, x: number, y: number, z: number) => { + const b = vrm.humanoid?.getNormalizedBoneNode(name) + if (b) b.rotation.set(x, y, z) + } + set("leftUpperArm", 0, 0, 1.2) // Arm runter an die Seite + set("rightUpperArm", 0, 0, -1.2) + set("leftLowerArm", 0, -0.2, 0) // leichte Beugung + set("rightLowerArm", 0, 0.2, 0) + vrm.humanoid?.update() +} + +const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const +const EMO_TO_EXPR: Record = { + neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed", +} + +function VrmModel({ url, audioLevel, emotion, onError }: { + url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void +}) { + const [vrm, setVrm] = useState(null) + const smooth = useRef>({}) + const blink = useRef({ t: 0, next: 3, active: 0 }) + + useEffect(() => { + let disposed = false + let loaded: VRM | null = null + const loader = new GLTFLoader() + loader.register((parser) => new VRMLoaderPlugin(parser)) + loader.load( + url, + (gltf) => { + if (disposed) return + const v = gltf.userData.vrm as VRM | undefined + if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return } + VRMUtils.removeUnnecessaryVertices(gltf.scene) + if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v) + v.scene.rotation.y = Math.PI // dem Betrachter zuwenden + applyRestPose(v) // T-Pose → entspannte A-Pose (Arme unten) + loaded = v + setVrm(v) + }, + undefined, + (err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") }, + ) + return () => { + disposed = true + if (loaded) VRMUtils.deepDispose(loaded.scene) + setVrm(null) + } + }, [url, onError]) + + useFrame((_, delta) => { + if (!vrm) return + const em = vrm.expressionManager + if (em) { + // Lippensync: 'aa' folgt geglättet dem Audiopegel. + const target = audioLevel.current?.current ?? 0 + const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6 + smooth.current.aa = aa + em.setValue("aa", aa) + + // Mimik: weich zur Ziel-Expression lerpen. + const want = EMO_TO_EXPR[emotion.current] + for (const name of EXPRESSIONS) { + const tv = want === name ? 0.75 : 0 + const cv = smooth.current[name] ?? 0 + const nv = cv + (tv - cv) * Math.min(1, delta * 4) + smooth.current[name] = nv + em.setValue(name, nv) + } + + // Blinzeln: kurzer Dreieckspuls alle 3–7 s. + const b = blink.current + b.t += delta + if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 } + let blinkVal = 0 + if (b.active > 0) { + b.active -= delta + const p = 1 - b.active / 0.16 // 0..1 Fortschritt + blinkVal = 1 - Math.abs(p - 0.5) * 2 // 0 → 1 → 0 + } + em.setValue("blink", Math.max(0, blinkVal)) + } + vrm.update(delta) + }) + + return vrm ? : null +} + +export function Avatar3D({ url, audioLevel, emotion }: { + url: string; audioLevel: LevelRef; emotion: EmotionRef +}) { + const [err, setErr] = useState(null) + return ( +
+ + + + + {/* key=url → bei Avatarwechsel sauber neu mounten */} + + + + {err && ( +
+ {err} +
+ )} +
+ ) +} diff --git a/frontend/src/components/voice/AvatarPicker.tsx b/frontend/src/components/voice/AvatarPicker.tsx new file mode 100644 index 0000000..1a74616 --- /dev/null +++ b/frontend/src/components/voice/AvatarPicker.tsx @@ -0,0 +1,184 @@ +import { useEffect, useRef, useState } from "react" +import { Upload, Link2, ExternalLink, Check, Sparkles } from "lucide-react" +import { saveUploadedVrm, loadUploadedVrm } from "@/lib/voice/vrmStore" + +// Avatar selbst aussuchen: kuratierte Galerie (öffentliche, CORS-freie VRMs) + eigenes .vrm +// hochladen + per URL laden + VRoid-Hub-Link. Dazu die Stimm-Auswahl (Engine + Stimme). +// Auswahl bleibt in localStorage / IndexedDB erhalten. + +interface GalleryItem { id: string; label: string; url: string; note?: string } + +// Verifiziert: 200 + Access-Control-Allow-Origin:* (im Browser ladbar). +const GALLERY: GalleryItem[] = [ + { id: "sample-a", label: "VRoid Sample A", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_A.vrm", note: "Anime, weiblich" }, + { id: "sample-b", label: "VRoid Sample B", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_B.vrm", note: "Anime, männlich" }, + { id: "pixiv", label: "Pixiv Demo", url: "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm", note: "VRM1-Testmodell" }, +] + +export const DEFAULT_AVATAR = GALLERY[0].url + +interface Voice { engine: string; id: string; label: string; clonable?: boolean } + +export function AvatarPicker({ avatarUrl, onAvatarChange }: { + avatarUrl: string; onAvatarChange: (url: string) => void +}) { + const [urlInput, setUrlInput] = useState("") + const [voices, setVoices] = useState([]) + const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper") + const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "") + const fileRef = useRef(null) + + // Hochgeladenes VRM nach Reload wiederherstellen. + useEffect(() => { + if (localStorage.getItem("mc_voice_avatar_uploaded") === "1") { + loadUploadedVrm().then((buf) => { + if (buf) onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" }))) + }) + } + }, [onAvatarChange]) + + // Stimmen vom Sidecar holen. + useEffect(() => { + fetch("/api/voice/voices") + .then((r) => (r.ok ? r.json() : Promise.reject())) + .then((d) => setVoices(d.voices || [])) + .catch(() => setVoices([])) + }, []) + + const pickGallery = (url: string) => { + localStorage.setItem("mc_voice_avatar_url", url) + localStorage.removeItem("mc_voice_avatar_uploaded") + onAvatarChange(url) + } + + const onUpload = async (file: File) => { + const buf = await file.arrayBuffer() + await saveUploadedVrm(buf) + localStorage.setItem("mc_voice_avatar_uploaded", "1") + localStorage.removeItem("mc_voice_avatar_url") + onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" }))) + } + + const loadUrl = () => { + const u = urlInput.trim() + if (u) pickGallery(u) + } + + const saveVoice = (eng: string, v: string) => { + setEngine(eng); setVoice(v) + localStorage.setItem("mc_voice_engine", eng) + localStorage.setItem("mc_voice_voice", v) + } + + const enginesAvail = Array.from(new Set(voices.map((v) => v.engine))) + const voicesForEngine = voices.filter((v) => v.engine === engine) + + return ( +
+ {/* Galerie */} +
+
Avatar
+
+ {GALLERY.map((g) => { + const active = avatarUrl === g.url + return ( + + ) + })} +
+
+ + {/* Eigenes Modell */} +
+ { const f = e.target.files?.[0]; if (f) void onUpload(f) }} + /> + +
+ setUrlInput(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && loadUrl()} + placeholder="…oder VRM-URL einfügen" + className="min-w-0 flex-1 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50" + /> + +
+ + Mehr Avatare auf VRoid Hub (kostenlos) → herunterladen & hochladen + +
+ + {/* Stimme */} +
+
+ Stimme +
+ {voices.length === 0 ? ( +
+ Voice-Dienst nicht erreichbar — Stimmen werden geladen, sobald der Sidecar läuft. +
+ ) : ( +
+
+ {enginesAvail.map((eng) => ( + + ))} +
+ + {engine === "chatterbox" && ( +

+ Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Natürlichste Stimme + Voice-Cloning. +

+ )} +
+ )} +
+
+ ) +} diff --git a/frontend/src/lib/voice/audio.ts b/frontend/src/lib/voice/audio.ts new file mode 100644 index 0000000..98eb75f --- /dev/null +++ b/frontend/src/lib/voice/audio.ts @@ -0,0 +1,85 @@ +// Sequentielle Audio-Wiedergabe für die TTS-Antworten + Pegel-Messung fürs Lippensync. +// +// Die einzelnen Satz-WAVs kommen nacheinander rein (satzweise Synthese → niedrige Latenz). +// Wir spielen sie über EINEN AudioContext geordnet ab und hängen einen AnalyserNode dazwischen, +// dessen Energie pro Frame in `level.current` (0..1) landet — der 3D-Avatar liest das im +// useFrame und öffnet den Mund entsprechend. Kein Re-Render pro Frame (Mutable-Ref-Muster). + +export class AudioQueue { + private ctx: AudioContext + private analyser: AnalyserNode + private queue: ArrayBuffer[] = [] + private playing = false + private raf = 0 + private freq: Uint8Array + /** Mutable, vom Avatar pro Frame gelesen. 0 = Mund zu, 1 = weit offen. */ + readonly level = { current: 0 } + onSpeaking?: (speaking: boolean) => void + + constructor() { + const Ctor = window.AudioContext || (window as any).webkitAudioContext + this.ctx = new Ctor() + this.analyser = this.ctx.createAnalyser() + this.analyser.fftSize = 256 + this.analyser.smoothingTimeConstant = 0.6 + this.analyser.connect(this.ctx.destination) + this.freq = new Uint8Array(new ArrayBuffer(this.analyser.frequencyBinCount)) + } + + async enqueue(buf: ArrayBuffer) { + this.queue.push(buf) + if (!this.playing) await this.playNext() + } + + /** Laufende + wartende Wiedergabe verwerfen (z.B. wenn der Nutzer dazwischenredet). */ + clear() { + this.queue = [] + } + + private async playNext(): Promise { + const buf = this.queue.shift() + if (!buf) { + this.playing = false + this.stopMeter() + this.onSpeaking?.(false) + return + } + this.playing = true + this.onSpeaking?.(true) + if (this.ctx.state === "suspended") { + try { await this.ctx.resume() } catch { /* vom User-Gesture freigeschaltet */ } + } + let audioBuf: AudioBuffer + try { + audioBuf = await this.ctx.decodeAudioData(buf.slice(0)) + } catch { + return this.playNext() // kaputtes Segment überspringen + } + const src = this.ctx.createBufferSource() + src.buffer = audioBuf + src.connect(this.analyser) + src.onended = () => { void this.playNext() } + src.start() + this.startMeter() + } + + private startMeter() { + cancelAnimationFrame(this.raf) + const tick = () => { + this.analyser.getByteFrequencyData(this.freq) + // Sprachenergie liegt v.a. in den unteren/mittleren Bändern. + const n = Math.min(this.freq.length, 48) + let sum = 0 + for (let i = 2; i < n; i++) sum += this.freq[i] + const avg = sum / (n - 2) / 255 + this.level.current = Math.min(1, avg * 1.9) + this.raf = requestAnimationFrame(tick) + } + tick() + } + + private stopMeter() { + cancelAnimationFrame(this.raf) + this.level.current = 0 + } +} diff --git a/frontend/src/lib/voice/sentiment.ts b/frontend/src/lib/voice/sentiment.ts new file mode 100644 index 0000000..993fbed --- /dev/null +++ b/frontend/src/lib/voice/sentiment.ts @@ -0,0 +1,17 @@ +// Leichtgewichtige Stimmungs-Heuristik (v1) → treibt die Avatar-Mimik. +// Bewusst simpel/regelbasiert (kein Modell): mappt deutschen Antworttext auf eine VRM-Expression. +// Spätere Stufe: echte Hermes-Emotion/Audio-Tags. Siehe Plan. + +export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed" + +const RULES: [Emotion, RegExp][] = [ + ["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha|:\)|😊|😄|🎉)/i], + ["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i], + ["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i], + ["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i], +] + +export function sentimentToEmotion(text: string): Emotion { + for (const [emo, rx] of RULES) if (rx.test(text)) return emo + return "neutral" +} diff --git a/frontend/src/lib/voice/usePushToTalk.ts b/frontend/src/lib/voice/usePushToTalk.ts new file mode 100644 index 0000000..e68a792 --- /dev/null +++ b/frontend/src/lib/voice/usePushToTalk.ts @@ -0,0 +1,50 @@ +import { useCallback, useEffect, useRef, useState } from "react" + +// Push-to-talk-Aufnahme über MediaRecorder. `start` beim Drücken (Taste/Button), `stop` beim +// Loslassen → fertiges Audio-Blob (webm/opus) geht an `onAudio`. Bewusst minimal & generisch. +export function usePushToTalk(onAudio: (blob: Blob) => void) { + const [recording, setRecording] = useState(false) + const recRef = useRef(null) + const chunksRef = useRef([]) + const streamRef = useRef(null) + + const start = useCallback(async () => { + if (recRef.current) return + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (e) { + console.error("Mikrofon-Zugriff verweigert:", e) + return + } + streamRef.current = stream + const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") + ? "audio/webm;codecs=opus" + : "audio/webm" + const rec = new MediaRecorder(stream, { mimeType: mime }) + chunksRef.current = [] + rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) } + rec.onstop = () => { + const blob = new Blob(chunksRef.current, { type: mime }) + streamRef.current?.getTracks().forEach((t) => t.stop()) + streamRef.current = null + recRef.current = null + setRecording(false) + if (blob.size > 1200) onAudio(blob) // Mini-Blobs (Versehen) ignorieren + } + rec.start() + recRef.current = rec + setRecording(true) + }, [onAudio]) + + const stop = useCallback(() => { + recRef.current?.stop() + }, []) + + useEffect(() => () => { + recRef.current?.stop() + streamRef.current?.getTracks().forEach((t) => t.stop()) + }, []) + + return { recording, start, stop } +} diff --git a/frontend/src/lib/voice/useVoiceAgent.ts b/frontend/src/lib/voice/useVoiceAgent.ts new file mode 100644 index 0000000..4e2be3f --- /dev/null +++ b/frontend/src/lib/voice/useVoiceAgent.ts @@ -0,0 +1,193 @@ +import { useCallback, useEffect, useRef, useState } from "react" +import { usePushToTalk } from "./usePushToTalk" +import { AudioQueue } from "./audio" +import { sentimentToEmotion, type Emotion } from "./sentiment" + +// Orchestriert die ganze Voll-Duplex-Schleife im Browser: +// PTT-Audio → /api/voice/stt → User-Text +// → /api/voice/chat (SSE vom Hermes-Agenten, server-seitiger Verlauf via Session-Id) +// → Antwort satzweise schneiden → /api/voice/tts je Satz → AudioQueue (Abspielen + Lippensync) +// → Stimmung aus dem Antworttext → Avatar-Mimik +// +// Avatar-Anbindung ohne Re-Render: `audioLevel` (Mundöffnung) und `emotion` sind Mutable-Refs, +// die der 3D-Avatar pro Frame liest. + +export type VoiceStatus = "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error" +export interface ChatMsg { role: "user" | "assistant"; text: string } + +const SYSTEM_PROMPT = + "Du sprichst per Sprache mit dem Nutzer. Antworte natürlich, freundlich und KNAPP in ganzen, " + + "gut vorlesbaren Sätzen. Kein Markdown, keine Codeblöcke, keine Aufzählungszeichen, keine Emojis — " + + "reiner Fließtext, den man laut vorlesen kann." + +function getSessionId(): string { + let id = localStorage.getItem("mc_voice_session") + if (!id) { + id = "voice-" + Math.random().toString(36).slice(2) + Date.now().toString(36) + localStorage.setItem("mc_voice_session", id) + } + return id +} + +function readSettings() { + return { + engine: localStorage.getItem("mc_voice_engine") || "piper", + voice: localStorage.getItem("mc_voice_voice") || "", + } +} + +// Zerlegt einen wachsenden Text-Stream in fertige Sätze. Gibt komplette Sätze zurück und behält +// den unvollständigen Rest. So kann das erste TTS schon starten, bevor die Antwort fertig ist. +function splitSentences(buffer: string): { sentences: string[]; rest: string } { + const sentences: string[] = [] + const rx = /[^.!?…]+[.!?…]+(\s|$)/g + let last = 0 + let m: RegExpExecArray | null + while ((m = rx.exec(buffer))) { + sentences.push(m[0].trim()) + last = rx.lastIndex + } + return { sentences, rest: buffer.slice(last) } +} + +export function useVoiceAgent() { + const [status, setStatus] = useState("idle") + const [messages, setMessages] = useState([]) + const [error, setError] = useState(null) + + const audioLevel = useRef({ current: 0 }) // wird gleich auf die Queue-Pegel gezeigt + const emotion = useRef("neutral") + const queueRef = useRef(null) + const sessionId = useRef(getSessionId()) + + // AudioQueue erst bei Bedarf (nach User-Geste) erzeugen — Autoplay-Policy. + const ensureQueue = useCallback(() => { + if (!queueRef.current) { + const q = new AudioQueue() + q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s)) + queueRef.current = q + audioLevel.current = q.level // Avatar liest ab jetzt echte Pegel + } + return queueRef.current + }, []) + + const handleAudio = useCallback(async (blob: Blob) => { + setError(null) + const queue = ensureQueue() + queue.clear() // evtl. laufende Antwort abbrechen (Barge-in) + + // 1) STT + setStatus("transcribing") + let userText = "" + try { + const fd = new FormData() + fd.append("audio", blob, "rec.webm") + const r = await fetch("/api/voice/stt", { method: "POST", body: fd }) + if (!r.ok) throw new Error(`STT ${r.status}`) + userText = (await r.json()).text?.trim() || "" + } catch (e: any) { + setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return + } + if (!userText) { setStatus("idle"); return } + setMessages((m) => [...m, { role: "user", text: userText }]) + + // 2) Chat (SSE) → 3) satzweises TTS + setStatus("thinking") + const { engine, voice } = readSettings() + let assistant = "" + let pending = "" + setMessages((m) => [...m, { role: "assistant", text: "" }]) + + const speak = (sentence: string) => { + if (!sentence.trim()) return + fetch("/api/voice/tts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: sentence, engine, voice }), + }) + .then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(`TTS ${r.status}`)))) + .then((buf) => queue.enqueue(buf)) + .catch((e) => console.error("TTS-Fehler:", e)) + } + + try { + const r = await fetch("/api/voice/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + text: userText, + session_id: sessionId.current, + system: SYSTEM_PROMPT, + }), + }) + if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`) + + const reader = r.body.getReader() + const dec = new TextDecoder() + let sse = "" + for (;;) { + const { done, value } = await reader.read() + if (done) break + sse += dec.decode(value, { stream: true }) + const events = sse.split("\n\n") + sse = events.pop() || "" + for (const ev of events) { + const line = ev.split("\n").find((l) => l.startsWith("data:")) + if (!line) continue + const data = line.slice(5).trim() + if (data === "[DONE]") continue + let json: any + try { json = JSON.parse(data) } catch { continue } + if (json.error) throw new Error(json.error) + const delta = json.choices?.[0]?.delta?.content || "" + if (!delta) continue + assistant += delta + pending += delta + emotion.current = sentimentToEmotion(assistant) + setMessages((m) => { + const copy = m.slice() + copy[copy.length - 1] = { role: "assistant", text: assistant } + return copy + }) + const { sentences, rest } = splitSentences(pending) + pending = rest + sentences.forEach(speak) + } + } + if (pending.trim()) speak(pending) // Rest (letzter Satz ohne Satzzeichen) + if (!assistant.trim()) setStatus("idle") + } catch (e: any) { + setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`) + } + }, [ensureQueue]) + + const { recording, start, stop } = usePushToTalk(handleAudio) + + const pressStart = useCallback(() => { + ensureQueue() + setStatus("listening") + void start() + }, [ensureQueue, start]) + + const pressEnd = useCallback(() => { stop() }, [stop]) + + const reset = useCallback(() => { + queueRef.current?.clear() + setMessages([]) + setError(null) + setStatus("idle") + localStorage.removeItem("mc_voice_session") + sessionId.current = getSessionId() + }, []) + + // Leerlauf-Status zurücksetzen, wenn nichts mehr spricht/aufnimmt. + useEffect(() => { + if (!recording && (status === "listening")) setStatus("transcribing") + }, [recording, status]) + + return { + status, messages, error, recording, + audioLevel, emotion, + pressStart, pressEnd, reset, + } +} diff --git a/frontend/src/lib/voice/vrmStore.ts b/frontend/src/lib/voice/vrmStore.ts new file mode 100644 index 0000000..804f365 --- /dev/null +++ b/frontend/src/lib/voice/vrmStore.ts @@ -0,0 +1,35 @@ +// Winziger IndexedDB-Wrapper, um ein hochgeladenes .vrm (ArrayBuffer) über Reloads hinweg zu +// behalten. localStorage scheidet aus (VRMs sind oft 5–40 MB). Ein einziger Record genügt. + +const DB = "mc2-voice" +const STORE = "avatar" +const KEY = "uploaded-vrm" + +function open(): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB, 1) + req.onupgradeneeded = () => req.result.createObjectStore(STORE) + req.onsuccess = () => resolve(req.result) + req.onerror = () => reject(req.error) + }) +} + +export async function saveUploadedVrm(buf: ArrayBuffer): Promise { + const db = await open() + await new Promise((resolve, reject) => { + const tx = db.transaction(STORE, "readwrite") + tx.objectStore(STORE).put(buf, KEY) + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error) + }) +} + +export async function loadUploadedVrm(): Promise { + const db = await open() + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE, "readonly") + const req = tx.objectStore(STORE).get(KEY) + req.onsuccess = () => resolve((req.result as ArrayBuffer) ?? null) + req.onerror = () => reject(req.error) + }) +} diff --git a/frontend/src/nav.ts b/frontend/src/nav.ts index 6dd132b..62e551b 100644 --- a/frontend/src/nav.ts +++ b/frontend/src/nav.ts @@ -6,10 +6,11 @@ import { Bot, TerminalSquare, HelpCircle, + Mic, type LucideIcon, } from "lucide-react" -export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "guide" +export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "voice" | "guide" export interface NavItem { id: ViewId @@ -26,6 +27,7 @@ export const NAV: NavItem[] = [ { id: "connect", label: "Verbinden", hint: "IDE-/Agent-Configs erzeugen", icon: Plug }, { id: "agent", label: "Hermes", hint: "Agent-Status & Verdrahtung", icon: Bot }, { id: "terminal", label: "Terminal", hint: "Interaktives Hermes-Agent-Terminal", icon: TerminalSquare }, + { id: "voice", label: "Sprechen", hint: "Mit Hermes per Sprache reden (3D-Avatar)", icon: Mic }, { id: "guide", label: "Anleitung", hint: "Einrichten & Vibe-Coding", icon: HelpCircle }, ] diff --git a/frontend/src/views/VoiceView.tsx b/frontend/src/views/VoiceView.tsx new file mode 100644 index 0000000..845d2b4 --- /dev/null +++ b/frontend/src/views/VoiceView.tsx @@ -0,0 +1,119 @@ +import { useCallback, useEffect, useRef, useState } from "react" +import { Mic, RotateCcw, Loader2, Volume2 } from "lucide-react" +import { Avatar3D } from "@/components/voice/Avatar3D" +import { AvatarPicker, DEFAULT_AVATAR } from "@/components/voice/AvatarPicker" +import { useVoiceAgent } from "@/lib/voice/useVoiceAgent" +import { cn } from "@/lib/utils" + +const STATUS_LABEL: Record = { + idle: "Bereit — halte zum Sprechen", + listening: "Höre zu …", + transcribing: "Verstehe …", + thinking: "Hermes denkt …", + speaking: "Hermes spricht …", + error: "Fehler", +} + +export function VoiceView() { + const [avatarUrl, setAvatarUrl] = useState( + () => localStorage.getItem("mc_voice_avatar_url") || DEFAULT_AVATAR, + ) + const { status, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset } = + useVoiceAgent() + + const holding = useRef(false) + const onAvatarChange = useCallback((url: string) => setAvatarUrl(url), []) + + // Push-to-talk per Leertaste (solange der Sprechen-Tab fokussiert ist und kein Eingabefeld aktiv). + useEffect(() => { + const isField = (el: EventTarget | null) => + el instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName) + const down = (e: KeyboardEvent) => { + if (e.code !== "Space" || e.repeat || holding.current || isField(e.target)) return + e.preventDefault(); holding.current = true; pressStart() + } + const up = (e: KeyboardEvent) => { + if (e.code !== "Space" || !holding.current) return + e.preventDefault(); holding.current = false; pressEnd() + } + window.addEventListener("keydown", down) + window.addEventListener("keyup", up) + return () => { window.removeEventListener("keydown", down); window.removeEventListener("keyup", up) } + }, [pressStart, pressEnd]) + + const speaking = status === "speaking" + const busy = status === "transcribing" || status === "thinking" + + return ( +
+ {/* Avatar-Bühne */} +
+
+ +
+ + {/* Status + Push-to-talk */} +
+
+ {busy && } + {speaking && } + {error || STATUS_LABEL[status]} +
+ + +
+ Halten zum Sprechen · Leertaste geht auch +
+
+
+ + {/* Seitenspalte: Einstellungen + Transcript */} +
+
+ +
+ +
+
+ Gespräch + +
+
+ {messages.length === 0 && ( +

+ Halte den Knopf (oder die Leertaste) und sprich. Hermes hört zu, denkt mit seinem + vollen Gedächtnis und seinen Werkzeugen — und antwortet hörbar. +

+ )} + {messages.map((m, i) => ( +
+ + {m.role === "user" ? "Du" : "Hermes"} + + {m.text || …} +
+ ))} +
+
+
+
+ ) +} diff --git a/voice_service/app.py b/voice_service/app.py new file mode 100644 index 0000000..24b5091 --- /dev/null +++ b/voice_service/app.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Voice-Sidecar für Mission Control 2.0 — lokales STT + gestuftes TTS für „Mit Hermes reden". + +WARUM ein eigener Dienst? Die ML-Stacks (faster-whisper, piper, chatterbox + torch) brauchen +ihr eigenes Python-3.12-venv — das MC2-Backend läuft auf Python 3.14 und kann sie nicht +importieren. Genau wie der Mem0-Sidecar (mem0_service/) kapselt dieser schlanke FastAPI-Dienst +die schwere Voice-Logik und exponiert sie auf localhost. MC2 (backend/routers/voice.py) proxyt +ihn nach außen; der Browser-Voice-Client (Frontend „Sprechen"-Tab) redet nie direkt mit ihm. + +Pipeline-Rolle: +- STT : faster-whisper (Default `medium`, int8, CPU, Sprache=de) — Mikro-Audio → Text. +- TTS : GESTUFT, Engine im Request wählbar (kein Lock-in): + * `piper` — schneller Standard, CPU, quasi-sofort, robustes Deutsch (thorsten). + * `chatterbox` — Premium/Wunschstimme (MIT), Voice-Cloning, dt. über Multilingual. + Lazy-Load (Modell erst beim ersten Aufruf) → Dienststart bleibt schnell. + Device via VOICE_CHATTERBOX_DEVICE (cpu | cuda); ROCm/iGPU (Strix Halo) + per HSA_OVERRIDE_GFX_VERSION=11.0.0 als späterer Umschalter. + +Läuft als systemd-User-Dienst (deploy/voice-service.service) im ~/.voice/venv (Python 3.12). +Bind: 127.0.0.1 (nur lokal; MC2 proxyt nach außen). +""" + +import io +import logging +import os +import tempfile +import wave +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.responses import Response +from pydantic import BaseModel + +log = logging.getLogger("voice_service") + +# --- Konfiguration (alles über Env überschreibbar; Defaults = Box-Stand) ---------- +PORT = int(os.environ.get("VOICE_PORT", "8650")) +# STT +STT_MODEL = os.environ.get("VOICE_STT_MODEL", "medium") # base|small|medium|large-v3 +STT_DEVICE = os.environ.get("VOICE_STT_DEVICE", "cpu") +STT_COMPUTE = os.environ.get("VOICE_STT_COMPUTE", "int8") # int8=CPU-schonend +STT_LANG = os.environ.get("VOICE_STT_LANG", "de") +# Piper (Default-TTS): Verzeichnis mit *.onnx (+ *.onnx.json) Stimmen +VOICES_DIR = Path(os.environ.get("VOICE_PIPER_DIR", str(Path(__file__).resolve().parent / "voices"))) +PIPER_DEFAULT = os.environ.get("VOICE_PIPER_DEFAULT", "de_DE-thorsten-medium") +# Chatterbox (Premium-TTS) +CHATTERBOX_DEVICE = os.environ.get("VOICE_CHATTERBOX_DEVICE", "cpu") +CHATTERBOX_LANG = os.environ.get("VOICE_CHATTERBOX_LANG", "de") +# Optionaler Referenz-WAV für Voice-Cloning (10 s Sprachprobe). Leer = Chatterbox-Standardstimme. +CHATTERBOX_REF = os.environ.get("VOICE_CHATTERBOX_REF", "") + + +# ================================================================================= +# STT — faster-whisper (lazy Singleton) +# ================================================================================= +_stt = None + + +def stt_model(): + global _stt + if _stt is None: + from faster_whisper import WhisperModel + log.info("Lade faster-whisper '%s' (%s/%s) …", STT_MODEL, STT_DEVICE, STT_COMPUTE) + _stt = WhisperModel(STT_MODEL, device=STT_DEVICE, compute_type=STT_COMPUTE) + return _stt + + +def transcribe(audio_bytes: bytes, suffix: str, language: str) -> str: + # PyAV (in faster-whisper) dekodiert webm/opus/ogg/wav am robustesten aus einer Datei. + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tf: + tf.write(audio_bytes) + tmp = tf.name + try: + segments, _info = stt_model().transcribe( + tmp, language=language or None, vad_filter=True, beam_size=5, + ) + return "".join(s.text for s in segments).strip() + finally: + try: + os.unlink(tmp) + except OSError: + pass + + +# ================================================================================= +# TTS — Piper (lazy, je Stimme gecacht) +# ================================================================================= +_piper: dict = {} + + +def piper_voice(name: str): + if name not in _piper: + from piper import PiperVoice + onnx = VOICES_DIR / f"{name}.onnx" + if not onnx.exists(): + raise HTTPException(404, f"Piper-Stimme '{name}' nicht gefunden ({onnx}).") + log.info("Lade Piper-Stimme '%s' …", name) + _piper[name] = PiperVoice.load(str(onnx)) + return _piper[name] + + +def piper_tts(text: str, voice: str) -> bytes: + v = piper_voice(voice or PIPER_DEFAULT) + buf = io.BytesIO() + with wave.open(buf, "wb") as wav: + # piper-tts 1.2.x: synthesize(text, wave_file) schreibt einen kompletten WAV-Stream. + v.synthesize(text, wav) + return buf.getvalue() + + +def piper_list() -> list[dict]: + if not VOICES_DIR.exists(): + return [] + return [ + {"engine": "piper", "id": p.stem, "label": p.stem, "clonable": False} + for p in sorted(VOICES_DIR.glob("*.onnx")) + ] + + +# ================================================================================= +# TTS — Chatterbox (lazy; Multilingual für Deutsch; optional Voice-Cloning) +# ================================================================================= +_chatterbox = None + + +def chatterbox_model(): + global _chatterbox + if _chatterbox is None: + log.info("Lade Chatterbox (Multilingual, device=%s) — einmalig, dauert kurz …", CHATTERBOX_DEVICE) + from chatterbox.mtl_tts import ChatterboxMultilingualTTS + _chatterbox = ChatterboxMultilingualTTS.from_pretrained(device=CHATTERBOX_DEVICE) + return _chatterbox + + +def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes: + import soundfile as sf + model = chatterbox_model() + kwargs = {"language_id": language or CHATTERBOX_LANG} + ref = ref_path or CHATTERBOX_REF + if ref and os.path.exists(ref): + kwargs["audio_prompt_path"] = ref # Zero-Shot Voice-Cloning aus Referenz + wav = model.generate(text, **kwargs) + # wav = torch.Tensor [1, N] @ model.sr → in WAV-Bytes serialisieren. + import numpy as np + arr = wav.squeeze(0).detach().cpu().numpy().astype(np.float32) + buf = io.BytesIO() + sf.write(buf, arr, int(model.sr), format="WAV", subtype="PCM_16") + return buf.getvalue() + + +def chatterbox_list() -> list[dict]: + # Chatterbox hat keine festen „Stimm-Dateien": Standardstimme + optionale Klon-Referenz. + items = [{"engine": "chatterbox", "id": "default", "label": "Chatterbox (Standard, dt.)", "clonable": True}] + if CHATTERBOX_REF and os.path.exists(CHATTERBOX_REF): + items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (geklonte Stimme)", "clonable": True}) + return items + + +# ================================================================================= +# FastAPI +# ================================================================================= +@asynccontextmanager +async def lifespan(_app: FastAPI): + logging.basicConfig(level=logging.INFO) + try: + stt_model() # STT beim Start vorwärmen (Modell aus HF-Cache laden) + log.info("Voice-Sidecar bereit (STT '%s', Piper-Dir %s).", STT_MODEL, VOICES_DIR) + except Exception: + log.exception("STT-Vorwärmen fehlgeschlagen (Dienst läuft, /health meldet Detail).") + yield + + +app = FastAPI(title="MC2 Voice Sidecar", lifespan=lifespan) + + +class TTSIn(BaseModel): + text: str + engine: str = "piper" # piper | chatterbox + voice: str = "" # Piper-Stimmname; bei Chatterbox: "default" | "clone" + language: str = "" # überschreibt Default-Sprache + ref_path: str = "" # optionaler Klon-Referenz-WAV (Chatterbox) + + +@app.get("/health") +def health() -> dict: + return {"ok": True, "stt_model": STT_MODEL, + "engines": ["piper", "chatterbox"], + "piper_voices": [v["id"] for v in piper_list()]} + + +@app.get("/voices") +def voices() -> dict: + return {"voices": piper_list() + chatterbox_list(), + "default": {"engine": "piper", "voice": PIPER_DEFAULT}} + + +@app.post("/stt") +async def stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict: + data = await audio.read() + if not data: + raise HTTPException(400, "Leeres Audio.") + suffix = Path(audio.filename or "rec.webm").suffix or ".webm" + text = transcribe(data, suffix, language or STT_LANG) + return {"text": text} + + +@app.post("/tts") +def tts(body: TTSIn) -> Response: + text = (body.text or "").strip() + if not text: + raise HTTPException(400, "Leerer Text.") + if body.engine == "chatterbox": + audio = chatterbox_tts(text, body.language, body.ref_path) + else: + audio = piper_tts(text, body.voice) + return Response(content=audio, media_type="audio/wav") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="127.0.0.1", port=PORT) diff --git a/voice_service/install.sh b/voice_service/install.sh new file mode 100644 index 0000000..adca95d --- /dev/null +++ b/voice_service/install.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Voice-Sidecar-Setup AUF DER BOX (User hitonabi, sudo-frei). Idempotent — Erstinstallation +# + Updates. Eigenes Python-3.12-venv (~/.voice/venv), weil torch/chatterbox/faster-whisper +# nicht ins 3.14-Backend-venv passen (analog ~/.mem0/venv). Wird von deploy/deploy.sh aufgerufen. +set -euo pipefail + +SRC="${MC2_SRC:-$HOME/mission-control-v2}" +VENV="$HOME/.voice/venv" +VOICES="$HOME/.voice/voices" +PY="${VOICE_PYTHON:-python3.12}" + +command -v "$PY" >/dev/null 2>&1 || PY=python3 # Fallback, falls python3.12 nicht im PATH + +# --- venv ------------------------------------------------------------------- +if [ ! -x "$VENV/bin/python" ]; then + echo "[voice] Erstelle venv ($PY) → $VENV" + "$PY" -m venv "$VENV" +fi +"$VENV/bin/python" -m pip install -q --upgrade pip + +# --- Kern-Deps (STT + Piper + Server) — required ---------------------------- +echo "[voice] Installiere Kern-Abhängigkeiten …" +"$VENV/bin/python" -m pip install -q -r "$SRC/voice_service/requirements.txt" + +# --- Chatterbox (Premium-TTS) — best-effort, CPU-torch (kein 2-GB-CUDA-Wheel) ---- +echo "[voice] Installiere Chatterbox (Premium-TTS, CPU-torch) — best-effort …" +if "$VENV/bin/python" -m pip install -q --index-url https://download.pytorch.org/whl/cpu torch torchaudio \ + && "$VENV/bin/python" -m pip install -q chatterbox-tts; then + echo "[voice] Chatterbox bereit." +else + echo "[voice] WARN: Chatterbox-Install fehlgeschlagen — Piper bleibt als Default-Stimme aktiv." +fi + +# --- Piper-Stimmen (Deutsch) nach ~/.voice/voices --------------------------- +mkdir -p "$VOICES" +BASE="https://huggingface.co/rhasspy/piper-voices/resolve/main/de/de_DE" +get_voice() { # $1=relpfad-ohne-endung $2=dateibasis + local rel="$1" name="$2" + for ext in onnx onnx.json; do + if [ ! -f "$VOICES/$name.$ext" ]; then + echo "[voice] Lade Piper-Stimme $name.$ext …" + curl -fsSL "$BASE/$rel/$name.$ext" -o "$VOICES/$name.$ext" \ + || echo "[voice] WARN: $name.$ext konnte nicht geladen werden." + fi + done +} +get_voice "thorsten/medium" "de_DE-thorsten-medium" # Default (schnell, klar) +get_voice "thorsten/high" "de_DE-thorsten-high" # natürlicher, etwas langsamer +get_voice "kerstin/low" "de_DE-kerstin-low" # weibliche Alternative + +echo "[voice] Fertig. Stimmen in $VOICES:" +ls -1 "$VOICES"/*.onnx 2>/dev/null || echo "[voice] WARN: keine Piper-Stimme vorhanden!" diff --git a/voice_service/requirements.txt b/voice_service/requirements.txt new file mode 100644 index 0000000..dab821d --- /dev/null +++ b/voice_service/requirements.txt @@ -0,0 +1,9 @@ +# Kern (immer nötig — STT + Piper-TTS + Server). Chatterbox + torch zieht install.sh +# separat best-effort nach (CPU-Wheel), damit der Voice-Loop auch ohne Premium-Engine läuft. +fastapi +uvicorn +python-multipart +soundfile +numpy +faster-whisper +piper-tts==1.2.0