Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
+2
-1
@@ -18,7 +18,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
from starlette.requests import Request
|
||||
|
||||
from config import FRONTEND_DIST, VERSION
|
||||
from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system
|
||||
from routers import agent, connect, gateway_proxy, health, maintenance, memory, models, routing, system, voice
|
||||
from services import warmer
|
||||
|
||||
# Zentrales Logging — Level via MC_LOG_LEVEL (INFO default). Eine Konfiguration
|
||||
@@ -68,6 +68,7 @@ app.include_router(system.router)
|
||||
app.include_router(connect.router)
|
||||
app.include_router(memory.router)
|
||||
app.include_router(agent.router)
|
||||
app.include_router(voice.router) # Sprache: STT/TTS-Proxy + Hermes-Agent-Chat (Voice-Tab)
|
||||
app.include_router(gateway_proxy.router) # OpenAI-kompatibler /v1-Gateway (model:auto)
|
||||
app.include_router(maintenance.router)
|
||||
|
||||
|
||||
@@ -60,6 +60,35 @@ GATEWAY_URL = os.environ.get("MC_GATEWAY_URL", f"http://127.0.0.1:{os.environ.ge
|
||||
# --- Hermes Agent (eigener Dienst auf der Box) -------------------------------
|
||||
# Gateway (OpenAI-API des Agenten) + interaktives Web-Terminal (ttyd → `hermes chat`).
|
||||
HERMES_API_URL = os.environ.get("HERMES_API_URL", "http://127.0.0.1:8642").rstrip("/")
|
||||
# API-Key der Hermes-`api_server`-Plattform (~/.hermes/.env: API_SERVER_KEY). Nötig für
|
||||
# /v1/chat/completions (Voice-Pipeline) — Bearer-Auth, sonst 401. Derselbe volle Agent
|
||||
# (Tools + geteiltes Mem0) wie CLI/Telegram, nur über HTTP.
|
||||
def _read_hermes_env(key: str) -> str:
|
||||
"""Liest einen Schlüssel aus ~/.hermes/.env (Fallback, falls nicht in der Prozess-Env).
|
||||
Der MC2-Dienst erbt die Hermes-Secrets sonst nicht."""
|
||||
try:
|
||||
env_path = Path(os.path.expanduser(os.environ.get("HERMES_HOME", "~/.hermes"))) / ".env"
|
||||
for line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith(f"{key}="):
|
||||
return line.split("=", 1)[1].strip().strip('"').strip("'")
|
||||
except OSError:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
HERMES_API_KEY = (
|
||||
os.environ.get("HERMES_API_KEY")
|
||||
or os.environ.get("API_SERVER_KEY")
|
||||
or _read_hermes_env("API_SERVER_KEY")
|
||||
)
|
||||
# Modellfeld im OpenAI-Request; die api_server-Plattform nutzt ihr konfiguriertes Hirn,
|
||||
# das Feld ist i.d.R. kosmetisch. Override via Env, falls die Plattform strikt prüft.
|
||||
HERMES_API_MODEL = os.environ.get("HERMES_API_MODEL", "hermes")
|
||||
|
||||
# --- Voice-Sidecar (STT faster-whisper + TTS Piper/Chatterbox) ---------------
|
||||
# Eigenes Python-3.12-venv (~/.voice/venv), analog Mem0-Sidecar. MC2 proxyt nach außen.
|
||||
VOICE_SERVICE_URL = os.environ.get("MC_VOICE_SERVICE_URL", "http://127.0.0.1:8650").rstrip("/")
|
||||
# Hermes-Terminal: ttyd-Web-Terminal der interaktiven Agent-CLI (Ersatz für AnythingLLM-Chat).
|
||||
# Wird in MC2 per iframe eingebettet (Terminal-Seite). Siehe deploy/hermes-terminal.service.
|
||||
HERMES_TERMINAL_URL = os.environ.get("MC_HERMES_TERMINAL_URL", "http://192.168.178.151:7681").rstrip("/")
|
||||
|
||||
@@ -14,7 +14,7 @@ from pydantic import BaseModel
|
||||
|
||||
import httpx
|
||||
|
||||
from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL
|
||||
from config import GATEWAY_URL, HERMES_API_URL, LLAMA_SWAP_URL, MEM0_SERVICE_URL, VOICE_SERVICE_URL
|
||||
from services import backup as backup_svc
|
||||
from services.agent import agent_status
|
||||
from services.gateway import gateway_reachable
|
||||
@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
# Nur diese User-Dienste dürfen neugestartet werden.
|
||||
ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui"}
|
||||
ALLOWED_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-webui", "mem0-service", "voice-service"}
|
||||
# Quelle für Self-Update (auf der Box ~/mission-control-v2).
|
||||
SOURCE_DIR = os.path.expanduser(os.environ.get("MC2_SOURCE_DIR", "~/mission-control-v2"))
|
||||
|
||||
@@ -45,6 +45,13 @@ def _mem0_reachable() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _voice_reachable() -> bool:
|
||||
try:
|
||||
return httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=2).status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@router.get("/system/services")
|
||||
def services() -> dict:
|
||||
"""Aggregierte Erreichbarkeit aller Stack-Dienste (für die Health-Anzeige)."""
|
||||
@@ -57,6 +64,7 @@ def services() -> dict:
|
||||
{"name": "Hermes-Gateway", "url": HERMES_API_URL, "ok": a["gateway_reachable"]},
|
||||
{"name": "Hermes-Terminal", "url": a["terminal_url"], "ok": a["terminal_reachable"]},
|
||||
{"name": "Mem0 (Gedächtnis)", "url": MEM0_SERVICE_URL, "ok": _mem0_reachable()},
|
||||
{"name": "Voice (STT/TTS)", "url": VOICE_SERVICE_URL, "ok": _voice_reachable()},
|
||||
],
|
||||
"links": {
|
||||
"engine_ui": f"{LLAMA_SWAP_URL}/ui",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Voice-Endpoints für „Mit Hermes reden" (Browser-Voice + 3D-Avatar).
|
||||
|
||||
Dünner Layer: STT/TTS werden zum Voice-Sidecar (:8650) geproxyt; der Chat geht an den
|
||||
Hermes-`api_server` (:8642, OpenAI-kompatibel) — denselben vollen Agenten mit Tools +
|
||||
geteiltem Mem0 wie CLI/Telegram. Mit stabilem `X-Hermes-Session-Id` hält die Plattform den
|
||||
Transcript server-seitig, daher schickt der Client je Turn nur die neue User-Nachricht.
|
||||
|
||||
LAN-only (kein Token in der 2.0-Phase), wie die übrigen MC2-Endpoints.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from config import HERMES_API_KEY, HERMES_API_MODEL, HERMES_API_URL, VOICE_SERVICE_URL
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
_TIMEOUT = httpx.Timeout(120.0, connect=5.0) # Chatterbox-TTS auf CPU darf dauern
|
||||
|
||||
|
||||
class TTSIn(BaseModel):
|
||||
text: str
|
||||
engine: str = "piper"
|
||||
voice: str = ""
|
||||
language: str = ""
|
||||
ref_path: str = ""
|
||||
|
||||
|
||||
class ChatIn(BaseModel):
|
||||
text: str # die neue User-Äußerung (STT-Ergebnis)
|
||||
session_id: str # stabiler Voice-Faden → server-seitiger Transcript
|
||||
session_key: str = "" # optional: Langzeit-Memory-Scope
|
||||
system: str = "" # optionaler ephemerer System-Prompt (z.B. „antworte knapp/gesprochen")
|
||||
model: str = ""
|
||||
|
||||
|
||||
@router.get("/voice/health")
|
||||
def voice_health() -> dict:
|
||||
"""Erreichbarkeit des Voice-Sidecars + ob der Hermes-API-Key gesetzt ist."""
|
||||
out: dict = {"sidecar": False, "hermes_key": bool(HERMES_API_KEY)}
|
||||
try:
|
||||
r = httpx.get(f"{VOICE_SERVICE_URL}/health", timeout=httpx.Timeout(5.0))
|
||||
out["sidecar"] = r.status_code == 200
|
||||
out["detail"] = r.json() if r.status_code == 200 else None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
out["error"] = str(exc)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/voice/voices")
|
||||
def voice_voices() -> dict:
|
||||
try:
|
||||
r = httpx.get(f"{VOICE_SERVICE_URL}/voices", timeout=httpx.Timeout(10.0))
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(502, f"Voice-Sidecar nicht erreichbar: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/stt")
|
||||
async def voice_stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict:
|
||||
"""Mikro-Audio → Text (Proxy auf Sidecar /stt)."""
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
files = {"audio": (audio.filename or "rec.webm", data, audio.content_type or "audio/webm")}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(f"{VOICE_SERVICE_URL}/stt", files=files, data={"language": language})
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"STT fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/tts")
|
||||
async def voice_tts(body: TTSIn) -> Response:
|
||||
"""Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(f"{VOICE_SERVICE_URL}/tts", json=body.model_dump())
|
||||
r.raise_for_status()
|
||||
return Response(content=r.content, media_type=r.headers.get("content-type", "audio/wav"))
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"TTS fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/chat")
|
||||
async def voice_chat(body: ChatIn) -> StreamingResponse:
|
||||
"""Neue User-Äußerung → Hermes-Agent (api_server, streamend). SSE wird 1:1 durchgereicht.
|
||||
|
||||
Mit `X-Hermes-Session-Id` hält die Plattform den Verlauf — wir senden nur die neue Nachricht.
|
||||
Auth per Bearer (API_SERVER_KEY); ohne Key liefert :8642 ein 401."""
|
||||
if not HERMES_API_KEY:
|
||||
raise HTTPException(503, "HERMES_API_KEY/API_SERVER_KEY nicht gesetzt — Agent-Auth fehlt.")
|
||||
|
||||
messages = []
|
||||
if body.system:
|
||||
messages.append({"role": "system", "content": body.system})
|
||||
messages.append({"role": "user", "content": body.text})
|
||||
payload = {"model": body.model or HERMES_API_MODEL, "messages": messages, "stream": True}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {HERMES_API_KEY}",
|
||||
"X-Hermes-Session-Id": body.session_id,
|
||||
}
|
||||
if body.session_key:
|
||||
headers["X-Hermes-Session-Key"] = body.session_key
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(None, connect=5.0)) as client:
|
||||
async with client.stream(
|
||||
"POST", f"{HERMES_API_URL}/v1/chat/completions", json=payload, headers=headers,
|
||||
) as r:
|
||||
if r.status_code != 200:
|
||||
detail = (await r.aread()).decode("utf-8", "replace")[:500]
|
||||
yield f"data: {{\"error\": \"Hermes {r.status_code}: {detail}\"}}\n\n".encode()
|
||||
return
|
||||
async for chunk in r.aiter_raw():
|
||||
yield chunk
|
||||
except httpx.HTTPError as exc:
|
||||
yield f"data: {{\"error\": \"Verbindung zu Hermes fehlgeschlagen: {exc}\"}}\n\n".encode()
|
||||
|
||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||
@@ -19,7 +19,7 @@ from services import catalog, discover, jobengine, llamaswap, system
|
||||
|
||||
# System-Dienste (root, via sudo -n NOPASSWD) vs. User-Dienste (systemctl --user).
|
||||
SYSTEM_SERVICES = {"llama-swap"}
|
||||
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service"}
|
||||
USER_SERVICES = {"mission-control-2", "hermes-gateway", "hermes-terminal", "mem0-service", "voice-service"}
|
||||
|
||||
# Engine-Update: lädt den neuesten Vulkan-Build (deploy/update-engine.sh, läuft als root).
|
||||
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
|
||||
@@ -31,6 +31,12 @@ else
|
||||
echo "WARN: ~/.mem0/venv fehlt — Mem0-Sidecar wird nicht gestartet (siehe Plan A1)."
|
||||
fi
|
||||
|
||||
# Voice-Sidecar (lokales STT + gestuftes TTS für „Mit Hermes reden") — eigenes Python-3.12-venv
|
||||
# (~/.voice/venv), weil torch/chatterbox/faster-whisper nicht ins 3.14-Backend-venv passen.
|
||||
# install.sh ist idempotent (venv + Deps + Piper-Stimmen). Best-effort: schlägt es fehl, läuft
|
||||
# der restliche Stack weiter (der Voice-Tab meldet den Sidecar dann als offline).
|
||||
bash "$SRC/voice_service/install.sh" || echo "WARN: Voice-Sidecar-Setup fehlgeschlagen — Voice-Tab bleibt offline."
|
||||
|
||||
# Hermes-Memory-Provider-Plugin (auto-lernen/Recall via Mem0-Sidecar) nach ~/.hermes/plugins/
|
||||
# spiegeln. Context-only (kein Tool-Loop). Aktivierung in ~/.hermes/config.yaml:
|
||||
# memory.memory_enabled: true + memory.provider: mc2-memory (einmalig, box-lokal).
|
||||
@@ -48,6 +54,8 @@ cp "$SRC/deploy/mission-control-2.service" "$HOME/.config/systemd/user/mission-c
|
||||
cp "$SRC/deploy/hermes-terminal.service" "$HOME/.config/systemd/user/hermes-terminal.service"
|
||||
# Mem0-Sidecar-Unit (nur wenn das venv existiert).
|
||||
[ -x "$HOME/.mem0/venv/bin/python" ] && cp "$SRC/deploy/mem0-service.service" "$HOME/.config/systemd/user/mem0-service.service"
|
||||
# Voice-Sidecar-Unit (nur wenn das venv existiert).
|
||||
[ -x "$HOME/.voice/venv/bin/python" ] && cp "$SRC/deploy/voice-service.service" "$HOME/.config/systemd/user/voice-service.service"
|
||||
# Tägliches Zustands-Backup (mem0 + Configs/Secrets) — Timer + oneshot-Service. Siehe docs/BACKUP.md.
|
||||
cp "$SRC/deploy/mc2-backup.service" "$HOME/.config/systemd/user/mc2-backup.service"
|
||||
cp "$SRC/deploy/mc2-backup.timer" "$HOME/.config/systemd/user/mc2-backup.timer"
|
||||
@@ -55,10 +63,13 @@ systemctl --user daemon-reload
|
||||
systemctl --user enable mission-control-2 >/dev/null 2>&1 || true
|
||||
systemctl --user enable hermes-terminal >/dev/null 2>&1 || true
|
||||
systemctl --user enable mem0-service >/dev/null 2>&1 || true
|
||||
systemctl --user enable voice-service >/dev/null 2>&1 || true
|
||||
systemctl --user enable --now mc2-backup.timer >/dev/null 2>&1 || true
|
||||
loginctl enable-linger "$USER" >/dev/null 2>&1 || true
|
||||
# Mem0-Sidecar VOR dem Backend (re)starten, damit /api/memory sofort bedient wird.
|
||||
[ -x "$HOME/.mem0/venv/bin/python" ] && systemctl --user restart mem0-service 2>/dev/null || true
|
||||
# Voice-Sidecar (re)starten (best-effort; Erststart lädt das STT-Modell vor).
|
||||
[ -x "$HOME/.voice/venv/bin/python" ] && systemctl --user restart voice-service 2>/dev/null || true
|
||||
systemctl --user restart mission-control-2
|
||||
command -v ttyd >/dev/null 2>&1 && systemctl --user restart hermes-terminal 2>/dev/null || true
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=MC2 Voice Sidecar — lokales STT (faster-whisper) + gestuftes TTS (Piper/Chatterbox)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
# Eigenes Python-3.12-venv (~/.voice/venv) — torch/chatterbox/faster-whisper passen nicht ins
|
||||
# 3.14-Backend-venv (analog mem0-service). Bind 127.0.0.1: nur lokal; MC2 proxyt nach außen.
|
||||
Type=simple
|
||||
WorkingDirectory=%h/mission-control-v2/voice_service
|
||||
Environment=VOICE_PORT=8650
|
||||
Environment=VOICE_STT_MODEL=medium
|
||||
Environment=VOICE_STT_LANG=de
|
||||
Environment=VOICE_PIPER_DIR=%h/.voice/voices
|
||||
Environment=VOICE_PIPER_DEFAULT=de_DE-thorsten-medium
|
||||
Environment=VOICE_CHATTERBOX_DEVICE=cpu
|
||||
Environment=VOICE_CHATTERBOX_LANG=de
|
||||
Environment=TOKENIZERS_PARALLELISM=false
|
||||
ExecStart=%h/.voice/venv/bin/python -m uvicorn app:app --host 127.0.0.1 --port 8650
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
Generated
+562
-47
@@ -8,6 +8,9 @@
|
||||
"name": "mission-control-2-frontend",
|
||||
"version": "2.0.0",
|
||||
"dependencies": {
|
||||
"@pixiv/three-vrm": "^3.4.0",
|
||||
"@react-three/drei": "^9.114.0",
|
||||
"@react-three/fiber": "^8.17.10",
|
||||
"@tanstack/react-query": "^5.101.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
@@ -16,13 +19,15 @@
|
||||
"react-dom": "^18.3.1",
|
||||
"reagraph": "^4.22.0",
|
||||
"recharts": "^3.9.0",
|
||||
"tailwind-merge": "^2.5.5"
|
||||
"tailwind-merge": "^2.5.5",
|
||||
"three": "^0.169.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^22.10.1",
|
||||
"@types/react": "^18.3.12",
|
||||
"@types/react-dom": "^18.3.1",
|
||||
"@types/three": "^0.169.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"tailwindcss": "^4.0.0",
|
||||
"typescript": "^5.6.3",
|
||||
@@ -320,12 +325,6 @@
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@dimforge/rapier3d-compat": {
|
||||
"version": "0.12.0",
|
||||
"resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz",
|
||||
"integrity": "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.25.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
|
||||
@@ -824,6 +823,154 @@
|
||||
"integrity": "sha512-Rp7ll8BHrKB3wXaRFKhrltwZl1CiXGdibPxuWXvqGnKTnv8fqa/nvftYNuSbf+pbJWKYCXdBtYTITdAUTGGh0Q==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@monogrid/gainmap-js": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@monogrid/gainmap-js/-/gainmap-js-3.4.0.tgz",
|
||||
"integrity": "sha512-2Z0FATFHaoYJ8b+Y4y4Hgfn3FRFwuU5zRrk+9dFWp4uGAdHGqVEdP7HP+gLA3X469KXHmfupJaUbKo1b/aDKIg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"promise-worker-transferable": "^1.0.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">= 0.159.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm/-/three-vrm-3.5.4.tgz",
|
||||
"integrity": "sha512-hY0MnmKVLUebFR9QN9vCRXo0rZmLpxjLVZAupQv5qDDGCKZuNxlOts9PqTkWlzhkMLLdhB48AF+P4PCy/DFOzA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/three-vrm-core": "3.5.4",
|
||||
"@pixiv/three-vrm-materials-hdr-emissive-multiplier": "3.5.4",
|
||||
"@pixiv/three-vrm-materials-mtoon": "3.5.4",
|
||||
"@pixiv/three-vrm-materials-v0compat": "3.5.4",
|
||||
"@pixiv/three-vrm-node-constraint": "3.5.4",
|
||||
"@pixiv/three-vrm-springbone": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-core": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-core/-/three-vrm-core-3.5.4.tgz",
|
||||
"integrity": "sha512-CgaxZ4qX6JmE2oKsOVGGlheT011qP4bBZGUaMJJ5iQINI999+cveIdISe459B+AMYD1dU5h1+xCtDthLQNH1Bg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrm-0.0": "3.5.4",
|
||||
"@pixiv/types-vrmc-vrm-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-materials-hdr-emissive-multiplier": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-hdr-emissive-multiplier/-/three-vrm-materials-hdr-emissive-multiplier-3.5.4.tgz",
|
||||
"integrity": "sha512-3BPZ42qW38cHhP+imqEnqTFsltvYkHA/t4VzWFdQ9sngWt0NiFAOotEsv8gZn1ZgKE9VRQezXne6s5aRWQlQqA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-materials-mtoon": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-mtoon/-/three-vrm-materials-mtoon-3.5.4.tgz",
|
||||
"integrity": "sha512-vLHt7IYZxlijbCMa5TRRf6gaQjA65F/d7oZBNfZW3XC8sBg57ZV0M/xBAJm2d6UwVmTbBd9NG9li/Y9yuupDQw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrm-0.0": "3.5.4",
|
||||
"@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-materials-v0compat": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-materials-v0compat/-/three-vrm-materials-v0compat-3.5.4.tgz",
|
||||
"integrity": "sha512-qRRtg8vYFBRJpsa3evruWqmXr0Gwd8uJXwxxUKuKtrIN0Hw0hJsRLHkg9E238Q8xRTnoycdkSgX4VvVknvtXBA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrm-0.0": "3.5.4",
|
||||
"@pixiv/types-vrmc-materials-mtoon-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-node-constraint": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-node-constraint/-/three-vrm-node-constraint-3.5.4.tgz",
|
||||
"integrity": "sha512-nyAghDrYNp0Z2siEaY2+th+FzZdDs9EJy8K576Mpw9cy3jwtCJuTHBQmGzKkrgoLaBfrso+3Tdfgr62AJV0m1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrmc-node-constraint-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/three-vrm-springbone": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/three-vrm-springbone/-/three-vrm-springbone-3.5.4.tgz",
|
||||
"integrity": "sha512-8KWA7vHU+OnW6XcZPnDuUrjjPFRzmv2yxcTMNCYQvBS547/Iqo0RLbQvpO/ppceLTNJFY2TGXAWJU67aeazWjw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pixiv/types-vrm-0.0": "3.5.4",
|
||||
"@pixiv/types-vrmc-springbone-1.0": "3.5.4",
|
||||
"@pixiv/types-vrmc-springbone-extended-collider-1.0": "3.5.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/@pixiv/types-vrm-0.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrm-0.0/-/types-vrm-0.0-3.5.4.tgz",
|
||||
"integrity": "sha512-g9VMPikJxKJ/XgnhXxnqj6ejVhQ9WwOEUp7KyyZHsqRnHcQfSqyZFTs9wkAMCbwCqHXX/lg9+9Ogj8KhxCkbXw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-materials-hdr-emissive-multiplier-1.0/-/types-vrmc-materials-hdr-emissive-multiplier-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-h9GEQ3q1VTylL/P40kJ8uoQhfUYY54NhTG6Xsnl4X0jf7oHh8MKXYnMMn903hXml6082Yjx1C6x3DKckdnophg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-materials-mtoon-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-materials-mtoon-1.0/-/types-vrmc-materials-mtoon-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-AU5sOcsbmcnzRPfgfk7fzKhk5lNm2MqrbvtAsSlhylVMdlzrmXqIfRuexABcB9k+ysTp49qIFioot9KRlQcDUw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-node-constraint-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-node-constraint-1.0/-/types-vrmc-node-constraint-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-RRbK5NNvZv4ewRELezueCiDB11FGkt4pdZR/UJ027DJPaNzd6rls2OEJ8weQ3OmgHZpbx/BtdpX2JWX5WZZd/Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-springbone-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-springbone-1.0/-/types-vrmc-springbone-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-NO7HTRBuWEe89Wo9BRI5hX1kWVkZzA4YWg9XALSTLdIT8HMiArL4NRse/CF91jNuLcTGfPEWlBJyq9SciAQxAg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-springbone-extended-collider-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-springbone-extended-collider-1.0/-/types-vrmc-springbone-extended-collider-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-uzJmcRh/iHYnZPtnq6N1+jb+T/ptYNCyFEoArkWGNXHSfPjTkMcheTQkL5iMNvTE6hqhyMRVMha9OmbvyLmmUQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pixiv/types-vrmc-vrm-1.0": {
|
||||
"version": "3.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@pixiv/types-vrmc-vrm-1.0/-/types-vrmc-vrm-1.0-3.5.4.tgz",
|
||||
"integrity": "sha512-pkjT4QXT/Hp6rcq8J8EFEHIldbtPGYWOTaWdoRGMA8/KzJI5PdekZ6AUYbnSP6ElfJ3BSZ01DxxpQZa/UNsFqA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@radix-ui/primitive": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.4.tgz",
|
||||
@@ -1205,12 +1352,236 @@
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-spring/three": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.5.tgz",
|
||||
"integrity": "sha512-RxIsCoQfUqOS3POmhVHa1wdWS0wyHAUway73uRLp3GAL5U2iYVNdnzQsep6M2NZ994BlW8TcKuMtQHUqOsy6WA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@react-spring/animated": "~9.7.5",
|
||||
"@react-spring/core": "~9.7.5",
|
||||
"@react-spring/shared": "~9.7.5",
|
||||
"@react-spring/types": "~9.7.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@react-three/fiber": ">=6.0",
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0",
|
||||
"three": ">=0.126"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-spring/three/node_modules/@react-spring/animated": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.5.tgz",
|
||||
"integrity": "sha512-Tqrwz7pIlsSDITzxoLS3n/v/YCUHQdOIKtOJf4yL6kYVSDTSmVK1LI1Q3M/uu2Sx4X3pIWF3xLUhlsA6SPNTNg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@react-spring/shared": "~9.7.5",
|
||||
"@react-spring/types": "~9.7.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-spring/three/node_modules/@react-spring/core": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.5.tgz",
|
||||
"integrity": "sha512-rmEqcxRcu7dWh7MnCcMXLvrf6/SDlSokLaLTxiPlAYi11nN3B5oiCUAblO72o+9z/87j2uzxa2Inm8UbLjXA+w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@react-spring/animated": "~9.7.5",
|
||||
"@react-spring/shared": "~9.7.5",
|
||||
"@react-spring/types": "~9.7.5"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/react-spring/donate"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-spring/three/node_modules/@react-spring/rafz": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.5.tgz",
|
||||
"integrity": "sha512-5ZenDQMC48wjUzPAm1EtwQ5Ot3bLIAwwqP2w2owG5KoNdNHpEJV263nGhCeKKmuA3vG2zLLOdu3or6kuDjA6Aw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@react-spring/three/node_modules/@react-spring/shared": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.5.tgz",
|
||||
"integrity": "sha512-wdtoJrhUeeyD/PP/zo+np2s1Z820Ohr/BbuVYv+3dVLW7WctoiN7std8rISoYoHpUXtbkpesSKuPIw/6U1w1Pw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@react-spring/rafz": "~9.7.5",
|
||||
"@react-spring/types": "~9.7.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-spring/three/node_modules/@react-spring/types": {
|
||||
"version": "9.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.5.tgz",
|
||||
"integrity": "sha512-HVj7LrZ4ReHWBimBvu2SKND3cDVUPWKLqRTmWe/fNY6o1owGOX0cAHbdPDTMelgBlVbrTKrre6lFkhqGZErK/g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@react-spring/types": {
|
||||
"version": "9.6.1",
|
||||
"resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.6.1.tgz",
|
||||
"integrity": "sha512-POu8Mk0hIU3lRXB3bGIGe4VHIwwDsQyoD1F394OK7STTiX9w4dG3cTLljjYswkQN+hDSHRrj4O36kuVa7KPU8Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@react-three/drei": {
|
||||
"version": "9.122.0",
|
||||
"resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.122.0.tgz",
|
||||
"integrity": "sha512-SEO/F/rBCTjlLez7WAlpys+iGe9hty4rNgjZvgkQeXFSiwqD4Hbk/wNHMAbdd8vprO2Aj81mihv4dF5bC7D0CA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.26.0",
|
||||
"@mediapipe/tasks-vision": "0.10.17",
|
||||
"@monogrid/gainmap-js": "^3.0.6",
|
||||
"@react-spring/three": "~9.7.5",
|
||||
"@use-gesture/react": "^10.3.1",
|
||||
"camera-controls": "^2.9.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"detect-gpu": "^5.0.56",
|
||||
"glsl-noise": "^0.0.0",
|
||||
"hls.js": "^1.5.17",
|
||||
"maath": "^0.10.8",
|
||||
"meshline": "^3.3.1",
|
||||
"react-composer": "^5.0.3",
|
||||
"stats-gl": "^2.2.8",
|
||||
"stats.js": "^0.17.0",
|
||||
"suspend-react": "^0.1.3",
|
||||
"three-mesh-bvh": "^0.7.8",
|
||||
"three-stdlib": "^2.35.6",
|
||||
"troika-three-text": "^0.52.0",
|
||||
"tunnel-rat": "^0.1.2",
|
||||
"utility-types": "^3.11.0",
|
||||
"zustand": "^5.0.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@react-three/fiber": "^8",
|
||||
"react": "^18",
|
||||
"react-dom": "^18",
|
||||
"three": ">=0.137"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"react-dom": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": {
|
||||
"version": "0.10.17",
|
||||
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
|
||||
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@react-three/drei/node_modules/zustand": {
|
||||
"version": "5.0.14",
|
||||
"resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.14.tgz",
|
||||
"integrity": "sha512-/8tAspM5LMPr28b3fwLYrtdj77ECpfZviaP75CMTnwO8ISyaE4GDIG/9rDDYq/cH9D2Xw2A2RXglLInmVBQB/g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/react": ">=18.0.0",
|
||||
"immer": ">=9.0.6",
|
||||
"react": ">=18.0.0",
|
||||
"use-sync-external-store": ">=1.2.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/react": {
|
||||
"optional": true
|
||||
},
|
||||
"immer": {
|
||||
"optional": true
|
||||
},
|
||||
"react": {
|
||||
"optional": true
|
||||
},
|
||||
"use-sync-external-store": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@react-three/fiber": {
|
||||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-8.18.0.tgz",
|
||||
"integrity": "sha512-FYZZqD0UUHUswKz3LQl2Z7H24AhD14XGTsIRw3SJaXUxyfVMi+1yiZGmqTcPt/CkPpdU7rrxqcyQ1zJE5DjvIQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.17.8",
|
||||
"@types/react-reconciler": "^0.26.7",
|
||||
"@types/webxr": "*",
|
||||
"base64-js": "^1.5.1",
|
||||
"buffer": "^6.0.3",
|
||||
"its-fine": "^1.0.6",
|
||||
"react-reconciler": "^0.27.0",
|
||||
"react-use-measure": "^2.1.7",
|
||||
"scheduler": "^0.21.0",
|
||||
"suspend-react": "^0.1.3",
|
||||
"zustand": "^3.7.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"expo": ">=43.0",
|
||||
"expo-asset": ">=8.4",
|
||||
"expo-file-system": ">=11.0",
|
||||
"expo-gl": ">=11.0",
|
||||
"react": ">=18 <19",
|
||||
"react-dom": ">=18 <19",
|
||||
"react-native": ">=0.64",
|
||||
"three": ">=0.133"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"expo": {
|
||||
"optional": true
|
||||
},
|
||||
"expo-asset": {
|
||||
"optional": true
|
||||
},
|
||||
"expo-file-system": {
|
||||
"optional": true
|
||||
},
|
||||
"expo-gl": {
|
||||
"optional": true
|
||||
},
|
||||
"react-dom": {
|
||||
"optional": true
|
||||
},
|
||||
"react-native": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@react-three/fiber/node_modules/scheduler": {
|
||||
"version": "0.21.0",
|
||||
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.21.0.tgz",
|
||||
"integrity": "sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@react-three/fiber/node_modules/zustand": {
|
||||
"version": "3.7.2",
|
||||
"resolved": "https://registry.npmjs.org/zustand/-/zustand-3.7.2.tgz",
|
||||
"integrity": "sha512-PIJDIZKtokhof+9+60cpockVOq05sJzHCriyvaLBmEJixseQ1a5Kdov6fWZfWOu5SK9c+FhH1jU0tntLxRJYMA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.7.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=16.8"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"react": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@reduxjs/toolkit": {
|
||||
"version": "2.12.0",
|
||||
"resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz",
|
||||
@@ -2150,17 +2521,17 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/three": {
|
||||
"version": "0.185.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/three/-/three-0.185.0.tgz",
|
||||
"integrity": "sha512-O2Uy8Cj4Nonr8dWUUbifMdPe8B0Mq7EdOHb89S4+kjUw/KhbjTZrUuYlrQ1bpUKG+EP9QJnN7qNxbHGlGoLHMA==",
|
||||
"version": "0.169.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/three/-/three-0.169.0.tgz",
|
||||
"integrity": "sha512-oan7qCgJBt03wIaK+4xPWclYRPG9wzcg7Z2f5T8xYTNEF95kh0t0lklxLLYBDo7gQiGLYzE6iF4ta7nXF2bcsw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@dimforge/rapier3d-compat": "~0.12.0",
|
||||
"@tweenjs/tween.js": "~23.1.3",
|
||||
"@types/stats.js": "*",
|
||||
"@types/webxr": ">=0.5.17",
|
||||
"@types/webxr": "*",
|
||||
"@webgpu/types": "*",
|
||||
"fflate": "~0.8.2",
|
||||
"meshoptimizer": "~1.1.1"
|
||||
"meshoptimizer": "~0.18.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/three/node_modules/fflate": {
|
||||
@@ -2220,6 +2591,12 @@
|
||||
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@webgpu/types": {
|
||||
"version": "0.1.71",
|
||||
"resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz",
|
||||
"integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/@yomguithereal/helpers": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@yomguithereal/helpers/-/helpers-1.1.1.tgz",
|
||||
@@ -2238,6 +2615,26 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/base64-js": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
|
||||
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.10.38",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.38.tgz",
|
||||
@@ -2294,6 +2691,30 @@
|
||||
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
|
||||
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.2.1"
|
||||
}
|
||||
},
|
||||
"node_modules/camera-controls": {
|
||||
"version": "2.10.1",
|
||||
"resolved": "https://registry.npmjs.org/camera-controls/-/camera-controls-2.10.1.tgz",
|
||||
@@ -2916,12 +3337,44 @@
|
||||
"graphology-types": ">=0.23.0"
|
||||
}
|
||||
},
|
||||
"node_modules/hls.js": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz",
|
||||
"integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/hold-event": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/hold-event/-/hold-event-0.2.0.tgz",
|
||||
"integrity": "sha512-rko5P1XgHzy4B0NR0xVHEpWPgj0i23f8Mf8qsOugd1CHvfLR0PyIyy+8TAQQA9v8qAa1OZ4XuCKk04rxmPGHNQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ieee754": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
|
||||
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/immediate": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
|
||||
"integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/immer": {
|
||||
"version": "10.2.0",
|
||||
"resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz",
|
||||
@@ -2941,6 +3394,12 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/is-promise": {
|
||||
"version": "2.2.2",
|
||||
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz",
|
||||
"integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -3010,6 +3469,15 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/lie": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
|
||||
"integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"immediate": "~3.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz",
|
||||
@@ -3314,6 +3782,16 @@
|
||||
"react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc"
|
||||
}
|
||||
},
|
||||
"node_modules/maath": {
|
||||
"version": "0.10.8",
|
||||
"resolved": "https://registry.npmjs.org/maath/-/maath-0.10.8.tgz",
|
||||
"integrity": "sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@types/three": ">=0.134.0",
|
||||
"three": ">=0.134.0"
|
||||
}
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "0.30.21",
|
||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
||||
@@ -3324,10 +3802,19 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/meshline": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz",
|
||||
"integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/meshoptimizer": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.1.1.tgz",
|
||||
"integrity": "sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g==",
|
||||
"version": "0.18.1",
|
||||
"resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-0.18.1.tgz",
|
||||
"integrity": "sha512-ZhoIoL7TNV4s5B6+rx5mC//fw8/POGyNxS/DZyCJeiZ12ScLfVwRE/GfsxwiTkMYYD5DmK2/JXnEVXqL4rF+Sw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mnemonist": {
|
||||
@@ -3463,6 +3950,16 @@
|
||||
"integrity": "sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/promise-worker-transferable": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/promise-worker-transferable/-/promise-worker-transferable-1.0.4.tgz",
|
||||
"integrity": "sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"is-promise": "^2.1.0",
|
||||
"lie": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/prop-types": {
|
||||
"version": "15.8.1",
|
||||
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
|
||||
@@ -3826,34 +4323,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/reagraph/node_modules/glodrei/node_modules/maath": {
|
||||
"version": "0.10.8",
|
||||
"resolved": "https://registry.npmjs.org/maath/-/maath-0.10.8.tgz",
|
||||
"integrity": "sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@types/three": ">=0.134.0",
|
||||
"three": ">=0.134.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reagraph/node_modules/glodrei/node_modules/meshline": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz",
|
||||
"integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"three": ">=0.137"
|
||||
}
|
||||
},
|
||||
"node_modules/reagraph/node_modules/glodrei/node_modules/three-mesh-bvh": {
|
||||
"version": "0.7.6",
|
||||
"resolved": "https://registry.npmjs.org/three-mesh-bvh/-/three-mesh-bvh-0.7.6.tgz",
|
||||
"integrity": "sha512-rCjsnxEqR9r1/C/lCqzGLS67NDty/S/eT6rAJfDvsanrIctTWdNoR4ZOGWewCB13h1QkVo2BpmC0wakj1+0m8A==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"three": ">= 0.151.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reagraph/node_modules/glodrei/node_modules/troika-three-text": {
|
||||
"version": "0.47.2",
|
||||
"resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.47.2.tgz",
|
||||
@@ -3904,6 +4373,12 @@
|
||||
"loose-envify": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reagraph/node_modules/three": {
|
||||
"version": "0.154.0",
|
||||
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/recharts": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/recharts/-/recharts-3.9.0.tgz",
|
||||
@@ -4126,11 +4601,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/three": {
|
||||
"version": "0.154.0",
|
||||
"resolved": "https://registry.npmjs.org/three/-/three-0.154.0.tgz",
|
||||
"integrity": "sha512-Uzz8C/5GesJzv8i+Y2prEMYUwodwZySPcNhuJUdsVMH2Yn4Nm8qlbQe6qRN5fOhg55XB0WiLfTPBxVHxpE60ug==",
|
||||
"version": "0.169.0",
|
||||
"resolved": "https://registry.npmjs.org/three/-/three-0.169.0.tgz",
|
||||
"integrity": "sha512-Ed906MA3dR4TS5riErd4QBsRGPcx+HBDX2O5yYE5GqJeFQTPU+M56Va/f/Oph9X7uZo3W3o4l2ZhBZ6f6qUv0w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/three-mesh-bvh": {
|
||||
"version": "0.7.8",
|
||||
"resolved": "https://registry.npmjs.org/three-mesh-bvh/-/three-mesh-bvh-0.7.8.tgz",
|
||||
"integrity": "sha512-BGEZTOIC14U0XIRw3tO4jY7IjP7n7v24nv9JXS1CyeVRWOCkcOMhRnmENUjuV39gktAw4Ofhr0OvIAiTspQrrw==",
|
||||
"deprecated": "Deprecated due to three.js version incompatibility. Please use v0.8.0, instead.",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"three": ">= 0.151.0"
|
||||
}
|
||||
},
|
||||
"node_modules/three-stdlib": {
|
||||
"version": "2.36.1",
|
||||
"resolved": "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.36.1.tgz",
|
||||
@@ -4171,6 +4656,36 @@
|
||||
"url": "https://github.com/sponsors/SuperchupuDev"
|
||||
}
|
||||
},
|
||||
"node_modules/troika-three-text": {
|
||||
"version": "0.52.4",
|
||||
"resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.52.4.tgz",
|
||||
"integrity": "sha512-V50EwcYGruV5rUZ9F4aNsrytGdKcXKALjEtQXIOBfhVoZU9VAqZNIoGQ3TMiooVqFAbR1w15T+f+8gkzoFzawg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bidi-js": "^1.0.2",
|
||||
"troika-three-utils": "^0.52.4",
|
||||
"troika-worker-utils": "^0.52.0",
|
||||
"webgl-sdf-generator": "1.1.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"three": ">=0.125.0"
|
||||
}
|
||||
},
|
||||
"node_modules/troika-three-text/node_modules/troika-worker-utils": {
|
||||
"version": "0.52.0",
|
||||
"resolved": "https://registry.npmjs.org/troika-worker-utils/-/troika-worker-utils-0.52.0.tgz",
|
||||
"integrity": "sha512-W1CpvTHykaPH5brv5VHLfQo9D1OYuo0cSBEUQFFT/nBUzM8iD6Lq2/tgG/f1OelbAS1WtaTPQzE5uM49egnngw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/troika-three-utils": {
|
||||
"version": "0.52.4",
|
||||
"resolved": "https://registry.npmjs.org/troika-three-utils/-/troika-three-utils-0.52.4.tgz",
|
||||
"integrity": "sha512-NORAStSVa/BDiG52Mfudk4j1FG4jC4ILutB3foPnfGbOeIs9+G5vZLa0pnmnaftZUGm4UwSoqEpWdqvC7zms3A==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"three": ">=0.125.0"
|
||||
}
|
||||
},
|
||||
"node_modules/troika-worker-utils": {
|
||||
"version": "0.47.2",
|
||||
"resolved": "https://registry.npmjs.org/troika-worker-utils/-/troika-worker-utils-0.47.2.tgz",
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pixiv/three-vrm": "^3.4.0",
|
||||
"@react-three/drei": "^9.114.0",
|
||||
"@react-three/fiber": "^8.17.10",
|
||||
"@tanstack/react-query": "^5.101.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
@@ -17,13 +20,15 @@
|
||||
"react-dom": "^18.3.1",
|
||||
"reagraph": "^4.22.0",
|
||||
"recharts": "^3.9.0",
|
||||
"tailwind-merge": "^2.5.5"
|
||||
"tailwind-merge": "^2.5.5",
|
||||
"three": "^0.169.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^22.10.1",
|
||||
"@types/react": "^18.3.12",
|
||||
"@types/react-dom": "^18.3.1",
|
||||
"@types/three": "^0.169.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"tailwindcss": "^4.0.0",
|
||||
"typescript": "^5.6.3",
|
||||
|
||||
@@ -8,6 +8,7 @@ import { ConnectView } from "@/views/ConnectView"
|
||||
import { MemoryView } from "@/views/MemoryView"
|
||||
import { AgentView } from "@/views/AgentView"
|
||||
import { TerminalView } from "@/views/TerminalView"
|
||||
import { VoiceView } from "@/views/VoiceView"
|
||||
import { GuideView } from "@/views/GuideView"
|
||||
import { Placeholder } from "@/views/Placeholder"
|
||||
import { SystemDrawer } from "@/components/SystemDrawer"
|
||||
@@ -191,8 +192,9 @@ export default function App() {
|
||||
{view === "memory" && <MemoryView />}
|
||||
{view === "agent" && <AgentView />}
|
||||
{view === "terminal" && <TerminalView />}
|
||||
{view === "voice" && <VoiceView />}
|
||||
{view === "guide" && <GuideView />}
|
||||
{!["dashboard", "models", "connect", "memory", "agent", "terminal", "guide"].includes(view) && (
|
||||
{!["dashboard", "models", "connect", "memory", "agent", "terminal", "voice", "guide"].includes(view) && (
|
||||
<Placeholder title={active.label} hint={active.hint} />
|
||||
)}
|
||||
</main>
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { Canvas, useFrame } from "@react-three/fiber"
|
||||
import { OrbitControls } from "@react-three/drei"
|
||||
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
||||
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
||||
import type { Emotion } from "@/lib/voice/sentiment"
|
||||
|
||||
// 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und
|
||||
// stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame).
|
||||
|
||||
type LevelRef = MutableRefObject<{ current: number }> // audioLevel.current.current = Pegel 0..1
|
||||
type EmotionRef = MutableRefObject<Emotion>
|
||||
|
||||
// VRMs laden in T-Pose (Bindepose, Arme waagerecht). Wir senken Ober-/Unterarme auf den
|
||||
// normalisierten Humanoid-Knoten zu einer ruhigen A-Pose ab — sieht sofort natürlich aus.
|
||||
// (Echte Idle-Animation wäre die spätere Stufe.)
|
||||
function applyRestPose(vrm: VRM) {
|
||||
const set = (name: any, x: number, y: number, z: number) => {
|
||||
const b = vrm.humanoid?.getNormalizedBoneNode(name)
|
||||
if (b) b.rotation.set(x, y, z)
|
||||
}
|
||||
set("leftUpperArm", 0, 0, 1.2) // Arm runter an die Seite
|
||||
set("rightUpperArm", 0, 0, -1.2)
|
||||
set("leftLowerArm", 0, -0.2, 0) // leichte Beugung
|
||||
set("rightLowerArm", 0, 0.2, 0)
|
||||
vrm.humanoid?.update()
|
||||
}
|
||||
|
||||
const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
|
||||
const EMO_TO_EXPR: Record<Emotion, string | null> = {
|
||||
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
|
||||
}
|
||||
|
||||
function VrmModel({ url, audioLevel, emotion, onError }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void
|
||||
}) {
|
||||
const [vrm, setVrm] = useState<VRM | null>(null)
|
||||
const smooth = useRef<Record<string, number>>({})
|
||||
const blink = useRef({ t: 0, next: 3, active: 0 })
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false
|
||||
let loaded: VRM | null = null
|
||||
const loader = new GLTFLoader()
|
||||
loader.register((parser) => new VRMLoaderPlugin(parser))
|
||||
loader.load(
|
||||
url,
|
||||
(gltf) => {
|
||||
if (disposed) return
|
||||
const v = gltf.userData.vrm as VRM | undefined
|
||||
if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return }
|
||||
VRMUtils.removeUnnecessaryVertices(gltf.scene)
|
||||
if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
|
||||
v.scene.rotation.y = Math.PI // dem Betrachter zuwenden
|
||||
applyRestPose(v) // T-Pose → entspannte A-Pose (Arme unten)
|
||||
loaded = v
|
||||
setVrm(v)
|
||||
},
|
||||
undefined,
|
||||
(err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") },
|
||||
)
|
||||
return () => {
|
||||
disposed = true
|
||||
if (loaded) VRMUtils.deepDispose(loaded.scene)
|
||||
setVrm(null)
|
||||
}
|
||||
}, [url, onError])
|
||||
|
||||
useFrame((_, delta) => {
|
||||
if (!vrm) return
|
||||
const em = vrm.expressionManager
|
||||
if (em) {
|
||||
// Lippensync: 'aa' folgt geglättet dem Audiopegel.
|
||||
const target = audioLevel.current?.current ?? 0
|
||||
const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6
|
||||
smooth.current.aa = aa
|
||||
em.setValue("aa", aa)
|
||||
|
||||
// Mimik: weich zur Ziel-Expression lerpen.
|
||||
const want = EMO_TO_EXPR[emotion.current]
|
||||
for (const name of EXPRESSIONS) {
|
||||
const tv = want === name ? 0.75 : 0
|
||||
const cv = smooth.current[name] ?? 0
|
||||
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
||||
smooth.current[name] = nv
|
||||
em.setValue(name, nv)
|
||||
}
|
||||
|
||||
// Blinzeln: kurzer Dreieckspuls alle 3–7 s.
|
||||
const b = blink.current
|
||||
b.t += delta
|
||||
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
||||
let blinkVal = 0
|
||||
if (b.active > 0) {
|
||||
b.active -= delta
|
||||
const p = 1 - b.active / 0.16 // 0..1 Fortschritt
|
||||
blinkVal = 1 - Math.abs(p - 0.5) * 2 // 0 → 1 → 0
|
||||
}
|
||||
em.setValue("blink", Math.max(0, blinkVal))
|
||||
}
|
||||
vrm.update(delta)
|
||||
})
|
||||
|
||||
return vrm ? <primitive object={vrm.scene} /> : null
|
||||
}
|
||||
|
||||
export function Avatar3D({ url, audioLevel, emotion }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef
|
||||
}) {
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
return (
|
||||
<div className="relative h-full w-full">
|
||||
<Canvas
|
||||
camera={{ position: [0, 1.35, 1.25], fov: 30 }}
|
||||
gl={{ alpha: true, antialias: true }}
|
||||
style={{ background: "transparent" }}
|
||||
>
|
||||
<ambientLight intensity={0.85} />
|
||||
<directionalLight position={[1, 2, 2]} intensity={1.1} />
|
||||
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
|
||||
{/* key=url → bei Avatarwechsel sauber neu mounten */}
|
||||
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} onError={setErr} />
|
||||
<OrbitControls
|
||||
target={[0, 1.3, 0]}
|
||||
enablePan={false}
|
||||
minDistance={0.7}
|
||||
maxDistance={3}
|
||||
minPolarAngle={Math.PI / 3}
|
||||
maxPolarAngle={Math.PI / 1.8}
|
||||
/>
|
||||
</Canvas>
|
||||
{err && (
|
||||
<div className="absolute inset-x-0 bottom-3 mx-auto w-fit rounded-md bg-red-500/15 border border-red-500/30 px-3 py-1.5 text-xs text-red-300">
|
||||
{err}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { Upload, Link2, ExternalLink, Check, Sparkles } from "lucide-react"
|
||||
import { saveUploadedVrm, loadUploadedVrm } from "@/lib/voice/vrmStore"
|
||||
|
||||
// Avatar selbst aussuchen: kuratierte Galerie (öffentliche, CORS-freie VRMs) + eigenes .vrm
|
||||
// hochladen + per URL laden + VRoid-Hub-Link. Dazu die Stimm-Auswahl (Engine + Stimme).
|
||||
// Auswahl bleibt in localStorage / IndexedDB erhalten.
|
||||
|
||||
interface GalleryItem { id: string; label: string; url: string; note?: string }
|
||||
|
||||
// Verifiziert: 200 + Access-Control-Allow-Origin:* (im Browser ladbar).
|
||||
const GALLERY: GalleryItem[] = [
|
||||
{ id: "sample-a", label: "VRoid Sample A", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_A.vrm", note: "Anime, weiblich" },
|
||||
{ id: "sample-b", label: "VRoid Sample B", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_B.vrm", note: "Anime, männlich" },
|
||||
{ id: "pixiv", label: "Pixiv Demo", url: "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm", note: "VRM1-Testmodell" },
|
||||
]
|
||||
|
||||
export const DEFAULT_AVATAR = GALLERY[0].url
|
||||
|
||||
interface Voice { engine: string; id: string; label: string; clonable?: boolean }
|
||||
|
||||
export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
avatarUrl: string; onAvatarChange: (url: string) => void
|
||||
}) {
|
||||
const [urlInput, setUrlInput] = useState("")
|
||||
const [voices, setVoices] = useState<Voice[]>([])
|
||||
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
|
||||
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
|
||||
const fileRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
// Hochgeladenes VRM nach Reload wiederherstellen.
|
||||
useEffect(() => {
|
||||
if (localStorage.getItem("mc_voice_avatar_uploaded") === "1") {
|
||||
loadUploadedVrm().then((buf) => {
|
||||
if (buf) onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
})
|
||||
}
|
||||
}, [onAvatarChange])
|
||||
|
||||
// Stimmen vom Sidecar holen.
|
||||
useEffect(() => {
|
||||
fetch("/api/voice/voices")
|
||||
.then((r) => (r.ok ? r.json() : Promise.reject()))
|
||||
.then((d) => setVoices(d.voices || []))
|
||||
.catch(() => setVoices([]))
|
||||
}, [])
|
||||
|
||||
const pickGallery = (url: string) => {
|
||||
localStorage.setItem("mc_voice_avatar_url", url)
|
||||
localStorage.removeItem("mc_voice_avatar_uploaded")
|
||||
onAvatarChange(url)
|
||||
}
|
||||
|
||||
const onUpload = async (file: File) => {
|
||||
const buf = await file.arrayBuffer()
|
||||
await saveUploadedVrm(buf)
|
||||
localStorage.setItem("mc_voice_avatar_uploaded", "1")
|
||||
localStorage.removeItem("mc_voice_avatar_url")
|
||||
onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
}
|
||||
|
||||
const loadUrl = () => {
|
||||
const u = urlInput.trim()
|
||||
if (u) pickGallery(u)
|
||||
}
|
||||
|
||||
const saveVoice = (eng: string, v: string) => {
|
||||
setEngine(eng); setVoice(v)
|
||||
localStorage.setItem("mc_voice_engine", eng)
|
||||
localStorage.setItem("mc_voice_voice", v)
|
||||
}
|
||||
|
||||
const enginesAvail = Array.from(new Set(voices.map((v) => v.engine)))
|
||||
const voicesForEngine = voices.filter((v) => v.engine === engine)
|
||||
|
||||
return (
|
||||
<div className="space-y-5 text-sm">
|
||||
{/* Galerie */}
|
||||
<div>
|
||||
<div className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">Avatar</div>
|
||||
<div className="space-y-1.5">
|
||||
{GALLERY.map((g) => {
|
||||
const active = avatarUrl === g.url
|
||||
return (
|
||||
<button
|
||||
key={g.id}
|
||||
onClick={() => pickGallery(g.url)}
|
||||
className={`flex w-full items-center justify-between rounded-md border px-3 py-2 text-left transition-colors ${
|
||||
active ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
<span>
|
||||
<span className="font-medium">{g.label}</span>
|
||||
{g.note && <span className="ml-2 text-[11px] text-muted-foreground">{g.note}</span>}
|
||||
</span>
|
||||
{active && <Check className="h-4 w-4" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Eigenes Modell */}
|
||||
<div className="space-y-2">
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept=".vrm,model/gltf-binary"
|
||||
className="hidden"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) void onUpload(f) }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => fileRef.current?.click()}
|
||||
className="flex w-full items-center gap-2 rounded-md border border-border/40 bg-background/40 px-3 py-2 hover:bg-accent transition-colors"
|
||||
>
|
||||
<Upload className="h-4 w-4" /> Eigenes .vrm hochladen
|
||||
</button>
|
||||
<div className="flex gap-1.5">
|
||||
<input
|
||||
value={urlInput}
|
||||
onChange={(e) => setUrlInput(e.target.value)}
|
||||
onKeyDown={(e) => e.key === "Enter" && loadUrl()}
|
||||
placeholder="…oder VRM-URL einfügen"
|
||||
className="min-w-0 flex-1 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
/>
|
||||
<button onClick={loadUrl} className="rounded-md border border-border/40 bg-background/40 px-2.5 hover:bg-accent" title="Laden">
|
||||
<Link2 className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
<a
|
||||
href="https://hub.vroid.com/en/characters"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
className="flex items-center gap-1.5 text-xs text-primary/80 hover:text-primary"
|
||||
>
|
||||
<ExternalLink className="h-3.5 w-3.5" /> Mehr Avatare auf VRoid Hub (kostenlos) → herunterladen & hochladen
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{/* Stimme */}
|
||||
<div>
|
||||
<div className="mb-2 flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||
<Sparkles className="h-3.5 w-3.5" /> Stimme
|
||||
</div>
|
||||
{voices.length === 0 ? (
|
||||
<div className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-300">
|
||||
Voice-Dienst nicht erreichbar — Stimmen werden geladen, sobald der Sidecar läuft.
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="flex gap-1.5">
|
||||
{enginesAvail.map((eng) => (
|
||||
<button
|
||||
key={eng}
|
||||
onClick={() => saveVoice(eng, "")}
|
||||
className={`flex-1 rounded-md border px-2.5 py-1.5 text-xs capitalize transition-colors ${
|
||||
engine === eng ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
{eng === "piper" ? "Piper (schnell)" : eng === "chatterbox" ? "Chatterbox (premium)" : eng}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<select
|
||||
value={voice}
|
||||
onChange={(e) => saveVoice(engine, e.target.value)}
|
||||
className="w-full rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
>
|
||||
<option value="">Standardstimme</option>
|
||||
{voicesForEngine.map((v) => (
|
||||
<option key={v.id} value={v.id}>{v.label}{v.clonable ? " · klonbar" : ""}</option>
|
||||
))}
|
||||
</select>
|
||||
{engine === "chatterbox" && (
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Natürlichste Stimme + Voice-Cloning.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Sequentielle Audio-Wiedergabe für die TTS-Antworten + Pegel-Messung fürs Lippensync.
|
||||
//
|
||||
// Die einzelnen Satz-WAVs kommen nacheinander rein (satzweise Synthese → niedrige Latenz).
|
||||
// Wir spielen sie über EINEN AudioContext geordnet ab und hängen einen AnalyserNode dazwischen,
|
||||
// dessen Energie pro Frame in `level.current` (0..1) landet — der 3D-Avatar liest das im
|
||||
// useFrame und öffnet den Mund entsprechend. Kein Re-Render pro Frame (Mutable-Ref-Muster).
|
||||
|
||||
export class AudioQueue {
|
||||
private ctx: AudioContext
|
||||
private analyser: AnalyserNode
|
||||
private queue: ArrayBuffer[] = []
|
||||
private playing = false
|
||||
private raf = 0
|
||||
private freq: Uint8Array<ArrayBuffer>
|
||||
/** Mutable, vom Avatar pro Frame gelesen. 0 = Mund zu, 1 = weit offen. */
|
||||
readonly level = { current: 0 }
|
||||
onSpeaking?: (speaking: boolean) => void
|
||||
|
||||
constructor() {
|
||||
const Ctor = window.AudioContext || (window as any).webkitAudioContext
|
||||
this.ctx = new Ctor()
|
||||
this.analyser = this.ctx.createAnalyser()
|
||||
this.analyser.fftSize = 256
|
||||
this.analyser.smoothingTimeConstant = 0.6
|
||||
this.analyser.connect(this.ctx.destination)
|
||||
this.freq = new Uint8Array(new ArrayBuffer(this.analyser.frequencyBinCount))
|
||||
}
|
||||
|
||||
async enqueue(buf: ArrayBuffer) {
|
||||
this.queue.push(buf)
|
||||
if (!this.playing) await this.playNext()
|
||||
}
|
||||
|
||||
/** Laufende + wartende Wiedergabe verwerfen (z.B. wenn der Nutzer dazwischenredet). */
|
||||
clear() {
|
||||
this.queue = []
|
||||
}
|
||||
|
||||
private async playNext(): Promise<void> {
|
||||
const buf = this.queue.shift()
|
||||
if (!buf) {
|
||||
this.playing = false
|
||||
this.stopMeter()
|
||||
this.onSpeaking?.(false)
|
||||
return
|
||||
}
|
||||
this.playing = true
|
||||
this.onSpeaking?.(true)
|
||||
if (this.ctx.state === "suspended") {
|
||||
try { await this.ctx.resume() } catch { /* vom User-Gesture freigeschaltet */ }
|
||||
}
|
||||
let audioBuf: AudioBuffer
|
||||
try {
|
||||
audioBuf = await this.ctx.decodeAudioData(buf.slice(0))
|
||||
} catch {
|
||||
return this.playNext() // kaputtes Segment überspringen
|
||||
}
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
src.onended = () => { void this.playNext() }
|
||||
src.start()
|
||||
this.startMeter()
|
||||
}
|
||||
|
||||
private startMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
const tick = () => {
|
||||
this.analyser.getByteFrequencyData(this.freq)
|
||||
// Sprachenergie liegt v.a. in den unteren/mittleren Bändern.
|
||||
const n = Math.min(this.freq.length, 48)
|
||||
let sum = 0
|
||||
for (let i = 2; i < n; i++) sum += this.freq[i]
|
||||
const avg = sum / (n - 2) / 255
|
||||
this.level.current = Math.min(1, avg * 1.9)
|
||||
this.raf = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
private stopMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
this.level.current = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Leichtgewichtige Stimmungs-Heuristik (v1) → treibt die Avatar-Mimik.
|
||||
// Bewusst simpel/regelbasiert (kein Modell): mappt deutschen Antworttext auf eine VRM-Expression.
|
||||
// Spätere Stufe: echte Hermes-Emotion/Audio-Tags. Siehe Plan.
|
||||
|
||||
export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed"
|
||||
|
||||
const RULES: [Emotion, RegExp][] = [
|
||||
["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha|:\)|😊|😄|🎉)/i],
|
||||
["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i],
|
||||
["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i],
|
||||
["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i],
|
||||
]
|
||||
|
||||
export function sentimentToEmotion(text: string): Emotion {
|
||||
for (const [emo, rx] of RULES) if (rx.test(text)) return emo
|
||||
return "neutral"
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Push-to-talk-Aufnahme über MediaRecorder. `start` beim Drücken (Taste/Button), `stop` beim
|
||||
// Loslassen → fertiges Audio-Blob (webm/opus) geht an `onAudio`. Bewusst minimal & generisch.
|
||||
export function usePushToTalk(onAudio: (blob: Blob) => void) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recRef = useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = useRef<Blob[]>([])
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recRef.current) return
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
console.error("Mikrofon-Zugriff verweigert:", e)
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
|
||||
? "audio/webm;codecs=opus"
|
||||
: "audio/webm"
|
||||
const rec = new MediaRecorder(stream, { mimeType: mime })
|
||||
chunksRef.current = []
|
||||
rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) }
|
||||
rec.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: mime })
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current = null
|
||||
recRef.current = null
|
||||
setRecording(false)
|
||||
if (blob.size > 1200) onAudio(blob) // Mini-Blobs (Versehen) ignorieren
|
||||
}
|
||||
rec.start()
|
||||
recRef.current = rec
|
||||
setRecording(true)
|
||||
}, [onAudio])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
recRef.current?.stop()
|
||||
}, [])
|
||||
|
||||
useEffect(() => () => {
|
||||
recRef.current?.stop()
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
}, [])
|
||||
|
||||
return { recording, start, stop }
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { usePushToTalk } from "./usePushToTalk"
|
||||
import { AudioQueue } from "./audio"
|
||||
import { sentimentToEmotion, type Emotion } from "./sentiment"
|
||||
|
||||
// Orchestriert die ganze Voll-Duplex-Schleife im Browser:
|
||||
// PTT-Audio → /api/voice/stt → User-Text
|
||||
// → /api/voice/chat (SSE vom Hermes-Agenten, server-seitiger Verlauf via Session-Id)
|
||||
// → Antwort satzweise schneiden → /api/voice/tts je Satz → AudioQueue (Abspielen + Lippensync)
|
||||
// → Stimmung aus dem Antworttext → Avatar-Mimik
|
||||
//
|
||||
// Avatar-Anbindung ohne Re-Render: `audioLevel` (Mundöffnung) und `emotion` sind Mutable-Refs,
|
||||
// die der 3D-Avatar pro Frame liest.
|
||||
|
||||
export type VoiceStatus = "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error"
|
||||
export interface ChatMsg { role: "user" | "assistant"; text: string }
|
||||
|
||||
const SYSTEM_PROMPT =
|
||||
"Du sprichst per Sprache mit dem Nutzer. Antworte natürlich, freundlich und KNAPP in ganzen, " +
|
||||
"gut vorlesbaren Sätzen. Kein Markdown, keine Codeblöcke, keine Aufzählungszeichen, keine Emojis — " +
|
||||
"reiner Fließtext, den man laut vorlesen kann."
|
||||
|
||||
function getSessionId(): string {
|
||||
let id = localStorage.getItem("mc_voice_session")
|
||||
if (!id) {
|
||||
id = "voice-" + Math.random().toString(36).slice(2) + Date.now().toString(36)
|
||||
localStorage.setItem("mc_voice_session", id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
function readSettings() {
|
||||
return {
|
||||
engine: localStorage.getItem("mc_voice_engine") || "piper",
|
||||
voice: localStorage.getItem("mc_voice_voice") || "",
|
||||
}
|
||||
}
|
||||
|
||||
// Zerlegt einen wachsenden Text-Stream in fertige Sätze. Gibt komplette Sätze zurück und behält
|
||||
// den unvollständigen Rest. So kann das erste TTS schon starten, bevor die Antwort fertig ist.
|
||||
function splitSentences(buffer: string): { sentences: string[]; rest: string } {
|
||||
const sentences: string[] = []
|
||||
const rx = /[^.!?…]+[.!?…]+(\s|$)/g
|
||||
let last = 0
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = rx.exec(buffer))) {
|
||||
sentences.push(m[0].trim())
|
||||
last = rx.lastIndex
|
||||
}
|
||||
return { sentences, rest: buffer.slice(last) }
|
||||
}
|
||||
|
||||
export function useVoiceAgent() {
|
||||
const [status, setStatus] = useState<VoiceStatus>("idle")
|
||||
const [messages, setMessages] = useState<ChatMsg[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const audioLevel = useRef({ current: 0 }) // wird gleich auf die Queue-Pegel gezeigt
|
||||
const emotion = useRef<Emotion>("neutral")
|
||||
const queueRef = useRef<AudioQueue | null>(null)
|
||||
const sessionId = useRef<string>(getSessionId())
|
||||
|
||||
// AudioQueue erst bei Bedarf (nach User-Geste) erzeugen — Autoplay-Policy.
|
||||
const ensureQueue = useCallback(() => {
|
||||
if (!queueRef.current) {
|
||||
const q = new AudioQueue()
|
||||
q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s))
|
||||
queueRef.current = q
|
||||
audioLevel.current = q.level // Avatar liest ab jetzt echte Pegel
|
||||
}
|
||||
return queueRef.current
|
||||
}, [])
|
||||
|
||||
const handleAudio = useCallback(async (blob: Blob) => {
|
||||
setError(null)
|
||||
const queue = ensureQueue()
|
||||
queue.clear() // evtl. laufende Antwort abbrechen (Barge-in)
|
||||
|
||||
// 1) STT
|
||||
setStatus("transcribing")
|
||||
let userText = ""
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", blob, "rec.webm")
|
||||
const r = await fetch("/api/voice/stt", { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`STT ${r.status}`)
|
||||
userText = (await r.json()).text?.trim() || ""
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return
|
||||
}
|
||||
if (!userText) { setStatus("idle"); return }
|
||||
setMessages((m) => [...m, { role: "user", text: userText }])
|
||||
|
||||
// 2) Chat (SSE) → 3) satzweises TTS
|
||||
setStatus("thinking")
|
||||
const { engine, voice } = readSettings()
|
||||
let assistant = ""
|
||||
let pending = ""
|
||||
setMessages((m) => [...m, { role: "assistant", text: "" }])
|
||||
|
||||
const speak = (sentence: string) => {
|
||||
if (!sentence.trim()) return
|
||||
fetch("/api/voice/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: sentence, engine, voice }),
|
||||
})
|
||||
.then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(`TTS ${r.status}`))))
|
||||
.then((buf) => queue.enqueue(buf))
|
||||
.catch((e) => console.error("TTS-Fehler:", e))
|
||||
}
|
||||
|
||||
try {
|
||||
const r = await fetch("/api/voice/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
text: userText,
|
||||
session_id: sessionId.current,
|
||||
system: SYSTEM_PROMPT,
|
||||
}),
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`)
|
||||
|
||||
const reader = r.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let sse = ""
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
sse += dec.decode(value, { stream: true })
|
||||
const events = sse.split("\n\n")
|
||||
sse = events.pop() || ""
|
||||
for (const ev of events) {
|
||||
const line = ev.split("\n").find((l) => l.startsWith("data:"))
|
||||
if (!line) continue
|
||||
const data = line.slice(5).trim()
|
||||
if (data === "[DONE]") continue
|
||||
let json: any
|
||||
try { json = JSON.parse(data) } catch { continue }
|
||||
if (json.error) throw new Error(json.error)
|
||||
const delta = json.choices?.[0]?.delta?.content || ""
|
||||
if (!delta) continue
|
||||
assistant += delta
|
||||
pending += delta
|
||||
emotion.current = sentimentToEmotion(assistant)
|
||||
setMessages((m) => {
|
||||
const copy = m.slice()
|
||||
copy[copy.length - 1] = { role: "assistant", text: assistant }
|
||||
return copy
|
||||
})
|
||||
const { sentences, rest } = splitSentences(pending)
|
||||
pending = rest
|
||||
sentences.forEach(speak)
|
||||
}
|
||||
}
|
||||
if (pending.trim()) speak(pending) // Rest (letzter Satz ohne Satzzeichen)
|
||||
if (!assistant.trim()) setStatus("idle")
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`)
|
||||
}
|
||||
}, [ensureQueue])
|
||||
|
||||
const { recording, start, stop } = usePushToTalk(handleAudio)
|
||||
|
||||
const pressStart = useCallback(() => {
|
||||
ensureQueue()
|
||||
setStatus("listening")
|
||||
void start()
|
||||
}, [ensureQueue, start])
|
||||
|
||||
const pressEnd = useCallback(() => { stop() }, [stop])
|
||||
|
||||
const reset = useCallback(() => {
|
||||
queueRef.current?.clear()
|
||||
setMessages([])
|
||||
setError(null)
|
||||
setStatus("idle")
|
||||
localStorage.removeItem("mc_voice_session")
|
||||
sessionId.current = getSessionId()
|
||||
}, [])
|
||||
|
||||
// Leerlauf-Status zurücksetzen, wenn nichts mehr spricht/aufnimmt.
|
||||
useEffect(() => {
|
||||
if (!recording && (status === "listening")) setStatus("transcribing")
|
||||
}, [recording, status])
|
||||
|
||||
return {
|
||||
status, messages, error, recording,
|
||||
audioLevel, emotion,
|
||||
pressStart, pressEnd, reset,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Winziger IndexedDB-Wrapper, um ein hochgeladenes .vrm (ArrayBuffer) über Reloads hinweg zu
|
||||
// behalten. localStorage scheidet aus (VRMs sind oft 5–40 MB). Ein einziger Record genügt.
|
||||
|
||||
const DB = "mc2-voice"
|
||||
const STORE = "avatar"
|
||||
const KEY = "uploaded-vrm"
|
||||
|
||||
function open(): Promise<IDBDatabase> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = indexedDB.open(DB, 1)
|
||||
req.onupgradeneeded = () => req.result.createObjectStore(STORE)
|
||||
req.onsuccess = () => resolve(req.result)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function saveUploadedVrm(buf: ArrayBuffer): Promise<void> {
|
||||
const db = await open()
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readwrite")
|
||||
tx.objectStore(STORE).put(buf, KEY)
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function loadUploadedVrm(): Promise<ArrayBuffer | null> {
|
||||
const db = await open()
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readonly")
|
||||
const req = tx.objectStore(STORE).get(KEY)
|
||||
req.onsuccess = () => resolve((req.result as ArrayBuffer) ?? null)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
+3
-1
@@ -6,10 +6,11 @@ import {
|
||||
Bot,
|
||||
TerminalSquare,
|
||||
HelpCircle,
|
||||
Mic,
|
||||
type LucideIcon,
|
||||
} from "lucide-react"
|
||||
|
||||
export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "guide"
|
||||
export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "voice" | "guide"
|
||||
|
||||
export interface NavItem {
|
||||
id: ViewId
|
||||
@@ -26,6 +27,7 @@ export const NAV: NavItem[] = [
|
||||
{ id: "connect", label: "Verbinden", hint: "IDE-/Agent-Configs erzeugen", icon: Plug },
|
||||
{ id: "agent", label: "Hermes", hint: "Agent-Status & Verdrahtung", icon: Bot },
|
||||
{ id: "terminal", label: "Terminal", hint: "Interaktives Hermes-Agent-Terminal", icon: TerminalSquare },
|
||||
{ id: "voice", label: "Sprechen", hint: "Mit Hermes per Sprache reden (3D-Avatar)", icon: Mic },
|
||||
{ id: "guide", label: "Anleitung", hint: "Einrichten & Vibe-Coding", icon: HelpCircle },
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { Mic, RotateCcw, Loader2, Volume2 } from "lucide-react"
|
||||
import { Avatar3D } from "@/components/voice/Avatar3D"
|
||||
import { AvatarPicker, DEFAULT_AVATAR } from "@/components/voice/AvatarPicker"
|
||||
import { useVoiceAgent } from "@/lib/voice/useVoiceAgent"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
idle: "Bereit — halte zum Sprechen",
|
||||
listening: "Höre zu …",
|
||||
transcribing: "Verstehe …",
|
||||
thinking: "Hermes denkt …",
|
||||
speaking: "Hermes spricht …",
|
||||
error: "Fehler",
|
||||
}
|
||||
|
||||
export function VoiceView() {
|
||||
const [avatarUrl, setAvatarUrl] = useState(
|
||||
() => localStorage.getItem("mc_voice_avatar_url") || DEFAULT_AVATAR,
|
||||
)
|
||||
const { status, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset } =
|
||||
useVoiceAgent()
|
||||
|
||||
const holding = useRef(false)
|
||||
const onAvatarChange = useCallback((url: string) => setAvatarUrl(url), [])
|
||||
|
||||
// Push-to-talk per Leertaste (solange der Sprechen-Tab fokussiert ist und kein Eingabefeld aktiv).
|
||||
useEffect(() => {
|
||||
const isField = (el: EventTarget | null) =>
|
||||
el instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName)
|
||||
const down = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || e.repeat || holding.current || isField(e.target)) return
|
||||
e.preventDefault(); holding.current = true; pressStart()
|
||||
}
|
||||
const up = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || !holding.current) return
|
||||
e.preventDefault(); holding.current = false; pressEnd()
|
||||
}
|
||||
window.addEventListener("keydown", down)
|
||||
window.addEventListener("keyup", up)
|
||||
return () => { window.removeEventListener("keydown", down); window.removeEventListener("keyup", up) }
|
||||
}, [pressStart, pressEnd])
|
||||
|
||||
const speaking = status === "speaking"
|
||||
const busy = status === "transcribing" || status === "thinking"
|
||||
|
||||
return (
|
||||
<div className="flex h-full gap-5">
|
||||
{/* Avatar-Bühne */}
|
||||
<div className="relative flex min-w-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/30 overflow-hidden">
|
||||
<div className="flex-1 min-h-0">
|
||||
<Avatar3D url={avatarUrl} audioLevel={audioLevel} emotion={emotion} />
|
||||
</div>
|
||||
|
||||
{/* Status + Push-to-talk */}
|
||||
<div className="shrink-0 flex flex-col items-center gap-3 border-t border-border/40 bg-background/30 py-5 backdrop-blur-sm">
|
||||
<div className={cn(
|
||||
"flex items-center gap-2 text-sm",
|
||||
status === "error" ? "text-red-400" : speaking ? "text-primary" : "text-muted-foreground",
|
||||
)}>
|
||||
{busy && <Loader2 className="h-4 w-4 animate-spin" />}
|
||||
{speaking && <Volume2 className="h-4 w-4 animate-pulse" />}
|
||||
<span>{error || STATUS_LABEL[status]}</span>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onPointerDown={(e) => { e.preventDefault(); holding.current = true; pressStart() }}
|
||||
onPointerUp={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
onPointerLeave={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
className={cn(
|
||||
"flex h-20 w-20 items-center justify-center rounded-full border-2 transition-all select-none touch-none",
|
||||
recording
|
||||
? "border-red-500 bg-red-500/20 scale-110 shadow-lg shadow-red-500/30"
|
||||
: "border-primary/60 bg-primary/15 hover:bg-primary/25 hover:scale-105",
|
||||
)}
|
||||
title="Gedrückt halten zum Sprechen (oder Leertaste halten)"
|
||||
>
|
||||
<Mic className={cn("h-8 w-8", recording ? "text-red-400" : "text-primary")} />
|
||||
</button>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Halten zum Sprechen · <kbd className="rounded bg-muted px-1 py-0.5 font-mono">Leertaste</kbd> geht auch
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Seitenspalte: Einstellungen + Transcript */}
|
||||
<div className="flex w-80 shrink-0 flex-col gap-4">
|
||||
<div className="rounded-xl border border-border/40 bg-card/40 p-4 overflow-y-auto scrollbar-thin max-h-[55%]">
|
||||
<AvatarPicker avatarUrl={avatarUrl} onAvatarChange={onAvatarChange} />
|
||||
</div>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/40">
|
||||
<div className="flex items-center justify-between border-b border-border/40 px-4 py-2.5">
|
||||
<span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Gespräch</span>
|
||||
<button onClick={reset} className="text-muted-foreground hover:text-foreground" title="Neues Gespräch">
|
||||
<RotateCcw className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex-1 space-y-2.5 overflow-y-auto p-4 scrollbar-thin">
|
||||
{messages.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Halte den Knopf (oder die Leertaste) und sprich. Hermes hört zu, denkt mit seinem
|
||||
vollen Gedächtnis und seinen Werkzeugen — und antwortet hörbar.
|
||||
</p>
|
||||
)}
|
||||
{messages.map((m, i) => (
|
||||
<div key={i} className={cn("text-sm", m.role === "user" ? "text-foreground" : "text-primary/90")}>
|
||||
<span className="mr-1.5 text-[10px] font-semibold uppercase text-muted-foreground">
|
||||
{m.role === "user" ? "Du" : "Hermes"}
|
||||
</span>
|
||||
{m.text || <span className="text-muted-foreground">…</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Voice-Sidecar für Mission Control 2.0 — lokales STT + gestuftes TTS für „Mit Hermes reden".
|
||||
|
||||
WARUM ein eigener Dienst? Die ML-Stacks (faster-whisper, piper, chatterbox + torch) brauchen
|
||||
ihr eigenes Python-3.12-venv — das MC2-Backend läuft auf Python 3.14 und kann sie nicht
|
||||
importieren. Genau wie der Mem0-Sidecar (mem0_service/) kapselt dieser schlanke FastAPI-Dienst
|
||||
die schwere Voice-Logik und exponiert sie auf localhost. MC2 (backend/routers/voice.py) proxyt
|
||||
ihn nach außen; der Browser-Voice-Client (Frontend „Sprechen"-Tab) redet nie direkt mit ihm.
|
||||
|
||||
Pipeline-Rolle:
|
||||
- STT : faster-whisper (Default `medium`, int8, CPU, Sprache=de) — Mikro-Audio → Text.
|
||||
- TTS : GESTUFT, Engine im Request wählbar (kein Lock-in):
|
||||
* `piper` — schneller Standard, CPU, quasi-sofort, robustes Deutsch (thorsten).
|
||||
* `chatterbox` — Premium/Wunschstimme (MIT), Voice-Cloning, dt. über Multilingual.
|
||||
Lazy-Load (Modell erst beim ersten Aufruf) → Dienststart bleibt schnell.
|
||||
Device via VOICE_CHATTERBOX_DEVICE (cpu | cuda); ROCm/iGPU (Strix Halo)
|
||||
per HSA_OVERRIDE_GFX_VERSION=11.0.0 als späterer Umschalter.
|
||||
|
||||
Läuft als systemd-User-Dienst (deploy/voice-service.service) im ~/.voice/venv (Python 3.12).
|
||||
Bind: 127.0.0.1 (nur lokal; MC2 proxyt nach außen).
|
||||
"""
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import wave
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
log = logging.getLogger("voice_service")
|
||||
|
||||
# --- Konfiguration (alles über Env überschreibbar; Defaults = Box-Stand) ----------
|
||||
PORT = int(os.environ.get("VOICE_PORT", "8650"))
|
||||
# STT
|
||||
STT_MODEL = os.environ.get("VOICE_STT_MODEL", "medium") # base|small|medium|large-v3
|
||||
STT_DEVICE = os.environ.get("VOICE_STT_DEVICE", "cpu")
|
||||
STT_COMPUTE = os.environ.get("VOICE_STT_COMPUTE", "int8") # int8=CPU-schonend
|
||||
STT_LANG = os.environ.get("VOICE_STT_LANG", "de")
|
||||
# Piper (Default-TTS): Verzeichnis mit *.onnx (+ *.onnx.json) Stimmen
|
||||
VOICES_DIR = Path(os.environ.get("VOICE_PIPER_DIR", str(Path(__file__).resolve().parent / "voices")))
|
||||
PIPER_DEFAULT = os.environ.get("VOICE_PIPER_DEFAULT", "de_DE-thorsten-medium")
|
||||
# Chatterbox (Premium-TTS)
|
||||
CHATTERBOX_DEVICE = os.environ.get("VOICE_CHATTERBOX_DEVICE", "cpu")
|
||||
CHATTERBOX_LANG = os.environ.get("VOICE_CHATTERBOX_LANG", "de")
|
||||
# Optionaler Referenz-WAV für Voice-Cloning (10 s Sprachprobe). Leer = Chatterbox-Standardstimme.
|
||||
CHATTERBOX_REF = os.environ.get("VOICE_CHATTERBOX_REF", "")
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# STT — faster-whisper (lazy Singleton)
|
||||
# =================================================================================
|
||||
_stt = None
|
||||
|
||||
|
||||
def stt_model():
|
||||
global _stt
|
||||
if _stt is None:
|
||||
from faster_whisper import WhisperModel
|
||||
log.info("Lade faster-whisper '%s' (%s/%s) …", STT_MODEL, STT_DEVICE, STT_COMPUTE)
|
||||
_stt = WhisperModel(STT_MODEL, device=STT_DEVICE, compute_type=STT_COMPUTE)
|
||||
return _stt
|
||||
|
||||
|
||||
def transcribe(audio_bytes: bytes, suffix: str, language: str) -> str:
|
||||
# PyAV (in faster-whisper) dekodiert webm/opus/ogg/wav am robustesten aus einer Datei.
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tf:
|
||||
tf.write(audio_bytes)
|
||||
tmp = tf.name
|
||||
try:
|
||||
segments, _info = stt_model().transcribe(
|
||||
tmp, language=language or None, vad_filter=True, beam_size=5,
|
||||
)
|
||||
return "".join(s.text for s in segments).strip()
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# TTS — Piper (lazy, je Stimme gecacht)
|
||||
# =================================================================================
|
||||
_piper: dict = {}
|
||||
|
||||
|
||||
def piper_voice(name: str):
|
||||
if name not in _piper:
|
||||
from piper import PiperVoice
|
||||
onnx = VOICES_DIR / f"{name}.onnx"
|
||||
if not onnx.exists():
|
||||
raise HTTPException(404, f"Piper-Stimme '{name}' nicht gefunden ({onnx}).")
|
||||
log.info("Lade Piper-Stimme '%s' …", name)
|
||||
_piper[name] = PiperVoice.load(str(onnx))
|
||||
return _piper[name]
|
||||
|
||||
|
||||
def piper_tts(text: str, voice: str) -> bytes:
|
||||
v = piper_voice(voice or PIPER_DEFAULT)
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as wav:
|
||||
# piper-tts 1.2.x: synthesize(text, wave_file) schreibt einen kompletten WAV-Stream.
|
||||
v.synthesize(text, wav)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def piper_list() -> list[dict]:
|
||||
if not VOICES_DIR.exists():
|
||||
return []
|
||||
return [
|
||||
{"engine": "piper", "id": p.stem, "label": p.stem, "clonable": False}
|
||||
for p in sorted(VOICES_DIR.glob("*.onnx"))
|
||||
]
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# TTS — Chatterbox (lazy; Multilingual für Deutsch; optional Voice-Cloning)
|
||||
# =================================================================================
|
||||
_chatterbox = None
|
||||
|
||||
|
||||
def chatterbox_model():
|
||||
global _chatterbox
|
||||
if _chatterbox is None:
|
||||
log.info("Lade Chatterbox (Multilingual, device=%s) — einmalig, dauert kurz …", CHATTERBOX_DEVICE)
|
||||
from chatterbox.mtl_tts import ChatterboxMultilingualTTS
|
||||
_chatterbox = ChatterboxMultilingualTTS.from_pretrained(device=CHATTERBOX_DEVICE)
|
||||
return _chatterbox
|
||||
|
||||
|
||||
def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes:
|
||||
import soundfile as sf
|
||||
model = chatterbox_model()
|
||||
kwargs = {"language_id": language or CHATTERBOX_LANG}
|
||||
ref = ref_path or CHATTERBOX_REF
|
||||
if ref and os.path.exists(ref):
|
||||
kwargs["audio_prompt_path"] = ref # Zero-Shot Voice-Cloning aus Referenz
|
||||
wav = model.generate(text, **kwargs)
|
||||
# wav = torch.Tensor [1, N] @ model.sr → in WAV-Bytes serialisieren.
|
||||
import numpy as np
|
||||
arr = wav.squeeze(0).detach().cpu().numpy().astype(np.float32)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, arr, int(model.sr), format="WAV", subtype="PCM_16")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def chatterbox_list() -> list[dict]:
|
||||
# Chatterbox hat keine festen „Stimm-Dateien": Standardstimme + optionale Klon-Referenz.
|
||||
items = [{"engine": "chatterbox", "id": "default", "label": "Chatterbox (Standard, dt.)", "clonable": True}]
|
||||
if CHATTERBOX_REF and os.path.exists(CHATTERBOX_REF):
|
||||
items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (geklonte Stimme)", "clonable": True})
|
||||
return items
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# FastAPI
|
||||
# =================================================================================
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
try:
|
||||
stt_model() # STT beim Start vorwärmen (Modell aus HF-Cache laden)
|
||||
log.info("Voice-Sidecar bereit (STT '%s', Piper-Dir %s).", STT_MODEL, VOICES_DIR)
|
||||
except Exception:
|
||||
log.exception("STT-Vorwärmen fehlgeschlagen (Dienst läuft, /health meldet Detail).")
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="MC2 Voice Sidecar", lifespan=lifespan)
|
||||
|
||||
|
||||
class TTSIn(BaseModel):
|
||||
text: str
|
||||
engine: str = "piper" # piper | chatterbox
|
||||
voice: str = "" # Piper-Stimmname; bei Chatterbox: "default" | "clone"
|
||||
language: str = "" # überschreibt Default-Sprache
|
||||
ref_path: str = "" # optionaler Klon-Referenz-WAV (Chatterbox)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"ok": True, "stt_model": STT_MODEL,
|
||||
"engines": ["piper", "chatterbox"],
|
||||
"piper_voices": [v["id"] for v in piper_list()]}
|
||||
|
||||
|
||||
@app.get("/voices")
|
||||
def voices() -> dict:
|
||||
return {"voices": piper_list() + chatterbox_list(),
|
||||
"default": {"engine": "piper", "voice": PIPER_DEFAULT}}
|
||||
|
||||
|
||||
@app.post("/stt")
|
||||
async def stt(audio: UploadFile = File(...), language: str = Form(default="")) -> dict:
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
suffix = Path(audio.filename or "rec.webm").suffix or ".webm"
|
||||
text = transcribe(data, suffix, language or STT_LANG)
|
||||
return {"text": text}
|
||||
|
||||
|
||||
@app.post("/tts")
|
||||
def tts(body: TTSIn) -> Response:
|
||||
text = (body.text or "").strip()
|
||||
if not text:
|
||||
raise HTTPException(400, "Leerer Text.")
|
||||
if body.engine == "chatterbox":
|
||||
audio = chatterbox_tts(text, body.language, body.ref_path)
|
||||
else:
|
||||
audio = piper_tts(text, body.voice)
|
||||
return Response(content=audio, media_type="audio/wav")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host="127.0.0.1", port=PORT)
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# Voice-Sidecar-Setup AUF DER BOX (User hitonabi, sudo-frei). Idempotent — Erstinstallation
|
||||
# + Updates. Eigenes Python-3.12-venv (~/.voice/venv), weil torch/chatterbox/faster-whisper
|
||||
# nicht ins 3.14-Backend-venv passen (analog ~/.mem0/venv). Wird von deploy/deploy.sh aufgerufen.
|
||||
set -euo pipefail
|
||||
|
||||
SRC="${MC2_SRC:-$HOME/mission-control-v2}"
|
||||
VENV="$HOME/.voice/venv"
|
||||
VOICES="$HOME/.voice/voices"
|
||||
PY="${VOICE_PYTHON:-python3.12}"
|
||||
|
||||
command -v "$PY" >/dev/null 2>&1 || PY=python3 # Fallback, falls python3.12 nicht im PATH
|
||||
|
||||
# --- venv -------------------------------------------------------------------
|
||||
if [ ! -x "$VENV/bin/python" ]; then
|
||||
echo "[voice] Erstelle venv ($PY) → $VENV"
|
||||
"$PY" -m venv "$VENV"
|
||||
fi
|
||||
"$VENV/bin/python" -m pip install -q --upgrade pip
|
||||
|
||||
# --- Kern-Deps (STT + Piper + Server) — required ----------------------------
|
||||
echo "[voice] Installiere Kern-Abhängigkeiten …"
|
||||
"$VENV/bin/python" -m pip install -q -r "$SRC/voice_service/requirements.txt"
|
||||
|
||||
# --- Chatterbox (Premium-TTS) — best-effort, CPU-torch (kein 2-GB-CUDA-Wheel) ----
|
||||
echo "[voice] Installiere Chatterbox (Premium-TTS, CPU-torch) — best-effort …"
|
||||
if "$VENV/bin/python" -m pip install -q --index-url https://download.pytorch.org/whl/cpu torch torchaudio \
|
||||
&& "$VENV/bin/python" -m pip install -q chatterbox-tts; then
|
||||
echo "[voice] Chatterbox bereit."
|
||||
else
|
||||
echo "[voice] WARN: Chatterbox-Install fehlgeschlagen — Piper bleibt als Default-Stimme aktiv."
|
||||
fi
|
||||
|
||||
# --- Piper-Stimmen (Deutsch) nach ~/.voice/voices ---------------------------
|
||||
mkdir -p "$VOICES"
|
||||
BASE="https://huggingface.co/rhasspy/piper-voices/resolve/main/de/de_DE"
|
||||
get_voice() { # $1=relpfad-ohne-endung $2=dateibasis
|
||||
local rel="$1" name="$2"
|
||||
for ext in onnx onnx.json; do
|
||||
if [ ! -f "$VOICES/$name.$ext" ]; then
|
||||
echo "[voice] Lade Piper-Stimme $name.$ext …"
|
||||
curl -fsSL "$BASE/$rel/$name.$ext" -o "$VOICES/$name.$ext" \
|
||||
|| echo "[voice] WARN: $name.$ext konnte nicht geladen werden."
|
||||
fi
|
||||
done
|
||||
}
|
||||
get_voice "thorsten/medium" "de_DE-thorsten-medium" # Default (schnell, klar)
|
||||
get_voice "thorsten/high" "de_DE-thorsten-high" # natürlicher, etwas langsamer
|
||||
get_voice "kerstin/low" "de_DE-kerstin-low" # weibliche Alternative
|
||||
|
||||
echo "[voice] Fertig. Stimmen in $VOICES:"
|
||||
ls -1 "$VOICES"/*.onnx 2>/dev/null || echo "[voice] WARN: keine Piper-Stimme vorhanden!"
|
||||
@@ -0,0 +1,9 @@
|
||||
# Kern (immer nötig — STT + Piper-TTS + Server). Chatterbox + torch zieht install.sh
|
||||
# separat best-effort nach (CPU-Wheel), damit der Voice-Loop auch ohne Premium-Engine läuft.
|
||||
fastapi
|
||||
uvicorn
|
||||
python-multipart
|
||||
soundfile
|
||||
numpy
|
||||
faster-whisper
|
||||
piper-tts==1.2.0
|
||||
Reference in New Issue
Block a user