Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)

Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-27 23:26:58 +02:00
parent 2360ad173a
commit 8e7ce1b1d3
22 changed files with 1887 additions and 54 deletions
@@ -0,0 +1,139 @@
+import { Canvas, useFrame } from "@react-three/fiber"
+import { OrbitControls } from "@react-three/drei"
+import { useEffect, useRef, useState, type MutableRefObject } from "react"
+import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
+import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
+import type { Emotion } from "@/lib/voice/sentiment"
+
+// 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und
+// stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame).
+
+type LevelRef = MutableRefObject<{ current: number }>   // audioLevel.current.current = Pegel 0..1
+type EmotionRef = MutableRefObject<Emotion>
+
+// VRMs laden in T-Pose (Bindepose, Arme waagerecht). Wir senken Ober-/Unterarme auf den
+// normalisierten Humanoid-Knoten zu einer ruhigen A-Pose ab — sieht sofort natürlich aus.
+// (Echte Idle-Animation wäre die spätere Stufe.)
+function applyRestPose(vrm: VRM) {
+  const set = (name: any, x: number, y: number, z: number) => {
+    const b = vrm.humanoid?.getNormalizedBoneNode(name)
+    if (b) b.rotation.set(x, y, z)
+  }
+  set("leftUpperArm", 0, 0, 1.2)      // Arm runter an die Seite
+  set("rightUpperArm", 0, 0, -1.2)
+  set("leftLowerArm", 0, -0.2, 0)     // leichte Beugung
+  set("rightLowerArm", 0, 0.2, 0)
+  vrm.humanoid?.update()
+}
+
+const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
+const EMO_TO_EXPR: Record<Emotion, string | null> = {
+  neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
+}
+
+function VrmModel({ url, audioLevel, emotion, onError }: {
+  url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void
+}) {
+  const [vrm, setVrm] = useState<VRM | null>(null)
+  const smooth = useRef<Record<string, number>>({})
+  const blink = useRef({ t: 0, next: 3, active: 0 })
+
+  useEffect(() => {
+    let disposed = false
+    let loaded: VRM | null = null
+    const loader = new GLTFLoader()
+    loader.register((parser) => new VRMLoaderPlugin(parser))
+    loader.load(
+      url,
+      (gltf) => {
+        if (disposed) return
+        const v = gltf.userData.vrm as VRM | undefined
+        if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return }
+        VRMUtils.removeUnnecessaryVertices(gltf.scene)
+        if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
+        v.scene.rotation.y = Math.PI            // dem Betrachter zuwenden
+        applyRestPose(v)                        // T-Pose → entspannte A-Pose (Arme unten)
+        loaded = v
+        setVrm(v)
+      },
+      undefined,
+      (err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") },
+    )
+    return () => {
+      disposed = true
+      if (loaded) VRMUtils.deepDispose(loaded.scene)
+      setVrm(null)
+    }
+  }, [url, onError])
+
+  useFrame((_, delta) => {
+    if (!vrm) return
+    const em = vrm.expressionManager
+    if (em) {
+      // Lippensync: 'aa' folgt geglättet dem Audiopegel.
+      const target = audioLevel.current?.current ?? 0
+      const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6
+      smooth.current.aa = aa
+      em.setValue("aa", aa)
+
+      // Mimik: weich zur Ziel-Expression lerpen.
+      const want = EMO_TO_EXPR[emotion.current]
+      for (const name of EXPRESSIONS) {
+        const tv = want === name ? 0.75 : 0
+        const cv = smooth.current[name] ?? 0
+        const nv = cv + (tv - cv) * Math.min(1, delta * 4)
+        smooth.current[name] = nv
+        em.setValue(name, nv)
+      }
+
+      // Blinzeln: kurzer Dreieckspuls alle 3–7 s.
+      const b = blink.current
+      b.t += delta
+      if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
+      let blinkVal = 0
+      if (b.active > 0) {
+        b.active -= delta
+        const p = 1 - b.active / 0.16          // 0..1 Fortschritt
+        blinkVal = 1 - Math.abs(p - 0.5) * 2   // 0 → 1 → 0
+      }
+      em.setValue("blink", Math.max(0, blinkVal))
+    }
+    vrm.update(delta)
+  })
+
+  return vrm ? <primitive object={vrm.scene} /> : null
+}
+
+export function Avatar3D({ url, audioLevel, emotion }: {
+  url: string; audioLevel: LevelRef; emotion: EmotionRef
+}) {
+  const [err, setErr] = useState<string | null>(null)
+  return (
+    <div className="relative h-full w-full">
+      <Canvas
+        camera={{ position: [0, 1.35, 1.25], fov: 30 }}
+        gl={{ alpha: true, antialias: true }}
+        style={{ background: "transparent" }}
+      >
+        <ambientLight intensity={0.85} />
+        <directionalLight position={[1, 2, 2]} intensity={1.1} />
+        <directionalLight position={[-1, 1, -1]} intensity={0.4} />
+        {/* key=url → bei Avatarwechsel sauber neu mounten */}
+        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} onError={setErr} />
+        <OrbitControls
+          target={[0, 1.3, 0]}
+          enablePan={false}
+          minDistance={0.7}
+          maxDistance={3}
+          minPolarAngle={Math.PI / 3}
+          maxPolarAngle={Math.PI / 1.8}
+        />
+      </Canvas>
+      {err && (
+        <div className="absolute inset-x-0 bottom-3 mx-auto w-fit rounded-md bg-red-500/15 border border-red-500/30 px-3 py-1.5 text-xs text-red-300">
+          {err}
+        </div>
+      )}
+    </div>
+  )
+}