Lucy v2 (Etappe 1.1): echte Vokal-Mundformen + Blick ohne Schielen

- Lippensync: spektraler Schwerpunkt -> aa/ih/ou-Mundformen statt nur Pegel ("Mund auf/zu") - Augen blicken auf weit entfernten Punkt in Kamerarichtung -> kein Konvergenz-Schielen bei naher Kamera Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-30 18:41:18 +02:00
parent cb4d7102b5
commit ef1e7eb8e6
2 changed files with 360 additions and 0 deletions
@@ -0,0 +1,185 @@
+import { Canvas, useFrame } from "@react-three/fiber"
+import { OrbitControls } from "@react-three/drei"
+import { useEffect, useRef, useState, type MutableRefObject } from "react"
+import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
+import { Object3D, Vector3 } from "three"
+import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
+import type { Emotion } from "../lib/voice/sentiment"
+
+// 3D-Avatar (VRM): Lippensync (Mund folgt TTS-Pegel), Blinzeln, Mimik, lebendige Idle-/Sprech-Bewegung.
+// Liest Live-Werte aus Mutable-Refs (kein Re-Render pro Frame).
+
+type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }>
+type EmotionRef = MutableRefObject<Emotion>
+
+const REST: Record<string, [number, number, number]> = {
+  leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0],
+}
+function setBone(vrm: VRM, name: string, x: number, y: number, z: number) {
+  const b = vrm.humanoid?.getNormalizedBoneNode(name as any)
+  if (b) b.rotation.set(x, y, z)
+}
+function applyRestPose(vrm: VRM) {
+  for (const [name, r] of Object.entries(REST)) setBone(vrm, name, r[0], r[1], r[2])
+  vrm.humanoid?.update()
+}
+
+function applyIdle(vrm: VRM, t: number, level: number, lookYaw: number, lookPitch: number, lean: number, speak: number) {
+  const breathe = Math.sin(t * 1.6), sway = Math.sin(t * 0.45), weight = Math.sin(t * 0.32), weight2 = Math.sin(t * 0.21 + 1)
+  const emph = Math.min(1, level * 1.6)
+  // Sprech-Rhythmus: organischer Doppel-Takt -> Kopf nickt/betont auf der Stimme
+  const beat = Math.sin(t * 2.4) + Math.sin(t * 3.7) * 0.5
+  const nod = (emph * 0.05 + speak * 0.02) * beat + emph * Math.sin(t * 1.3) * 0.03
+  const tilt = Math.sin(t * 0.7 + 0.5) * 0.03 + speak * Math.sin(t * 0.9) * 0.045   // gelegentliches Kopf-Neigen
+  setBone(vrm, "hips", 0, weight * 0.06, weight * 0.04 + weight2 * 0.02)
+  setBone(vrm, "spine", breathe * 0.03 + lean * 0.08 + speak * emph * 0.03, sway * 0.03, -weight * 0.04)
+  setBone(vrm, "chest", breathe * 0.025 + lean * 0.025, sway * 0.022, weight2 * 0.012)
+  setBone(vrm, "upperChest", breathe * 0.018, sway * 0.01, 0)
+  setBone(vrm, "neck", lookPitch * 0.4 + nod * 0.5, lookYaw * 0.4, tilt * 0.5)
+  setBone(vrm, "head", lookPitch * 0.6 + nod + Math.sin(t * 0.6) * 0.02,
+          lookYaw * 0.6 + Math.sin(t * 0.27) * 0.035, tilt + Math.sin(t * 0.5) * 0.025)
+  // Arme: lebhafterer Schwung um die Ruhepose + dezente Sprech-Geste — SICHER (kleine Amplituden)
+  const armSwing = Math.sin(t * 0.8) * 0.05 + speak * Math.sin(t * 1.6) * 0.045
+  const lift = speak * emph * 0.07          // hebt die Arme beim Reden minimal an (mit den Händen reden)
+  setBone(vrm, "leftUpperArm", -lift, 0, 1.16 + armSwing + weight * 0.05)
+  setBone(vrm, "rightUpperArm", -lift, 0, -1.16 - armSwing + weight * 0.05)
+  setBone(vrm, "leftLowerArm", 0, -0.18 - Math.sin(t * 0.8) * 0.04 - speak * 0.07, 0)
+  setBone(vrm, "rightLowerArm", 0, 0.18 + Math.sin(t * 0.8) * 0.04 + speak * 0.07, 0)
+}
+
+const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
+const EMO_TO_EXPR: Record<Emotion, string | null> = {
+  neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
+}
+
+function VrmModel({ url, audioLevel, emotion, status, onError }: {
+  url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>; onError: (m: string) => void
+}) {
+  const [vrm, setVrm] = useState<VRM | null>(null)
+  const smooth = useRef<Record<string, number>>({})
+  const blink = useRef({ t: 0, next: 3, active: 0 })
+  const motion = useRef({ yaw: 0, pitch: 0, tYaw: 0, tPitch: 0, t: 0, next: 2.5, lean: 0 })
+  const think = useRef(0)   // 0..1 Nachdenk-Pose-Blend
+  const speak = useRef(0)   // 0..1 Sprech-Lebendigkeit
+  const gaze = useRef(new Object3D())   // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera)
+  const headPos = useRef(new Vector3())
+  const camPos = useRef(new Vector3())
+
+  useEffect(() => {
+    let disposed = false, loaded: VRM | null = null
+    const loader = new GLTFLoader()
+    loader.register((parser) => new VRMLoaderPlugin(parser))
+    loader.load(url, (gltf) => {
+      if (disposed) return
+      const v = gltf.userData.vrm as VRM | undefined
+      if (!v) { onError("Datei enthaelt kein gueltiges VRM-Modell."); return }
+      VRMUtils.removeUnnecessaryVertices(gltf.scene)
+      if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
+      v.scene.rotation.y = Math.PI
+      applyRestPose(v)
+      loaded = v; setVrm(v)
+    }, undefined, (err) => { console.error("VRM-Load:", err); onError("Avatar konnte nicht geladen werden.") })
+    return () => { disposed = true; if (loaded) VRMUtils.deepDispose(loaded.scene); setVrm(null) }
+  }, [url, onError])
+
+  useFrame((state, delta) => {
+    if (!vrm) return
+    const target = audioLevel.current?.current ?? 0
+    // Blick zur Kamera, aber auf einen WEIT entfernten Punkt in Kamerarichtung -> Augen laufen
+    // nicht zusammen (kein Schielen bei naher Kamera), wirken aber weiter "auf dich gerichtet".
+    if (vrm.lookAt) {
+      const headNode = vrm.humanoid?.getNormalizedBoneNode("head")
+      const hp = headPos.current
+      if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0)
+      const cp = camPos.current.copy(state.camera.position)
+      gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp)  // 4x = Blickpunkt weit dahinter
+      vrm.lookAt.target = gaze.current
+    }
+    const st = status.current
+    const spk = (speak.current += (((st === "speaking") ? 1 : 0) - speak.current) * Math.min(1, delta * 4))
+    const m = motion.current
+    m.t += delta
+    if (m.t > m.next) {
+      // beim Sprechen häufiger + eher zur Kamera (engagiert), sonst lebhaftes Umschauen
+      const range = spk > 0.5 ? 0.28 : 0.6
+      m.tYaw = (Math.random() - 0.5) * range; m.tPitch = (Math.random() - 0.5) * 0.3
+      m.t = 0; m.next = (spk > 0.5 ? 1.4 : 1.9) + Math.random() * 2.6
+    }
+    m.yaw += (m.tYaw - m.yaw) * Math.min(1, delta * 1.8)
+    m.pitch += (m.tPitch - m.pitch) * Math.min(1, delta * 1.8)
+    // Nachdenk-Blend hoch wenn 'thinking'/'transcribing', sonst runter
+    const wantThink = (st === "thinking" || st === "transcribing") ? 1 : 0
+    think.current += (wantThink - think.current) * Math.min(1, delta * 2.5)
+    const th = think.current
+    const attentive = st === "listening" ? 1 : 0
+    m.lean += ((Math.min(1, target * 1.6) + attentive * 0.5) - m.lean) * Math.min(1, delta * 3)
+    // Beim Nachdenken: Blick zur Seite/oben (pensiv, weg von der Kamera) statt zufälligem Umschauen
+    const tc = state.clock.elapsedTime
+    // Nachdenken: Blick zur Seite/leicht nach oben (pensiv, weg von der Kamera) + Kopf-Neigung.
+    // KEINE Arm-Bewegung (risikolos; die Hand-zum-Kinn-Pose liess sich blind nicht sauber treffen).
+    const yaw = m.yaw + th * (-0.38 + Math.sin(tc * 0.5) * 0.05)
+    const pitch = m.pitch + th * (-0.12)
+    applyIdle(vrm, tc, target * (1 - th * 0.5), yaw, pitch, m.lean, spk)
+    if (th > 0.02) {
+      // sanftes nachdenkliches Kopf-Neigen (nur Roll dazu, Idle-Kopfbewegung bleibt erhalten)
+      setBone(vrm, "head", pitch * 0.6 + Math.sin(tc * 0.6) * 0.015, yaw * 0.6 + Math.sin(tc * 0.27) * 0.025,
+              Math.sin(tc * 0.5) * 0.02 + th * 0.14)
+      vrm.humanoid?.update()
+    }
+
+    const em = vrm.expressionManager
+    if (em) {
+      // Lippensync v2: echte Vokal-Mundformen (aa/ih/ou) aus der Audio-Analyse statt nur "Mund auf".
+      // Fallback auf den Gesamtpegel (target) fuer aa, falls keine Mundform-Werte vorliegen.
+      const lv = audioLevel.current
+      const visTargets: Record<string, number> = {
+        aa: lv?.aa ?? target, ih: lv?.ih ?? 0, ou: lv?.ou ?? 0,
+      }
+      for (const v of ["aa", "ih", "ou"]) {
+        const cv = smooth.current[v] ?? 0
+        const nv = cv + (visTargets[v] - cv) * Math.min(1, delta * 14)  // schnell genug fuers Sprechen
+        smooth.current[v] = nv; em.setValue(v, nv)
+      }
+      const want = EMO_TO_EXPR[emotion.current]
+      for (const name of EXPRESSIONS) {
+        const tv = want === name ? 0.75 : 0
+        const cv = smooth.current[name] ?? 0
+        const nv = cv + (tv - cv) * Math.min(1, delta * 4)
+        smooth.current[name] = nv; em.setValue(name, nv)
+      }
+      const b = blink.current
+      b.t += delta
+      if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
+      let blinkVal = 0
+      if (b.active > 0) { b.active -= delta; const p = 1 - b.active / 0.16; blinkVal = 1 - Math.abs(p - 0.5) * 2 }
+      em.setValue("blink", Math.max(0, blinkVal))
+    }
+    vrm.update(delta)
+  })
+
+  return vrm ? <primitive object={vrm.scene} /> : null
+}
+
+export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: {
+  url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
+}) {
+  const [err, setErr] = useState<string | null>(null)
+  const statusRef = useRef(status); statusRef.current = status
+  return (
+    <div className="avatarCanvas" style={{ position: "relative", height: "100%", width: "100%" }}>
+      <Canvas camera={{ position: [0, 1.35, 1.25], fov: 30 }} gl={{ alpha: true, antialias: true }} style={{ background: "transparent" }}>
+        <ambientLight intensity={0.85} />
+        <directionalLight position={[1, 2, 2]} intensity={1.1} />
+        <directionalLight position={[-1, 1, -1]} intensity={0.4} />
+        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} onError={setErr} />
+        <OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
+          minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
+      </Canvas>
+      {err && (
+        <div style={{ position: "absolute", left: 0, right: 0, bottom: 12, margin: "0 auto", width: "fit-content",
+          borderRadius: 8, background: "rgba(239,68,68,0.15)", border: "1px solid rgba(239,68,68,0.3)",
+          padding: "6px 12px", fontSize: 12, color: "#fca5a5" }}>{err}</div>
+      )}
+    </div>
+  )
+}
@@ -0,0 +1,175 @@
+// Audio-Wiedergabe + Pegelmessung fuers Lippensync (Avatar liest level.current).
+// Zwei Wege: enqueue(WAV-Buffer) [z.B. Warmup] und playPcmStream(PCM16-Stream) [die Live-Antwort,
+// lueckenlos via Web-Audio-Scheduling, niedrige Time-to-first-audio].
+//
+// LIPPENSYNC v2: statt nur einem Pegel ("Mund auf/zu") leiten wir aus dem Frequenz-Spektrum
+// echte Vokal-MUNDFORMEN ab (level.aa/ih/ou). Idee: die Helligkeit des Klangs (spektraler
+// Schwerpunkt) verraet grob den Vokal — helle Laute (i/e) = breiter Mund, dunkle (o/u) = runder
+// Mund, mittig = offenes "a". Das ist nicht phonetisch exakt, sieht aber lebendig+passend aus.
+function smoothstep(a: number, b: number, x: number): number {
+  const t = Math.max(0, Math.min(1, (x - a) / (b - a)))
+  return t * t * (3 - 2 * t)
+}
+
+export class AudioQueue {
+  private ctx: AudioContext
+  private analyser: AnalyserNode
+  private gain: GainNode
+  private queue: ArrayBuffer[] = []
+  private playing = false
+  private raf = 0
+  private freq: Uint8Array
+  // aktiver PCM-Stream (fuer Barge-in/clear): geplante Quellen + Abbruchsignal
+  private streamSources: AudioBufferSourceNode[] = []
+  private streamCancelled = false
+  // current = Gesamt-Pegel (Aura/Bewegung); aa/ih/ou = Mundform-Gewichte fuer den Avatar
+  readonly level = { current: 0, aa: 0, ih: 0, ou: 0 }
+  onSpeaking?: (speaking: boolean) => void
+
+  static readVolume(): number {
+    const raw = localStorage.getItem("lucy_volume")
+    if (raw === null || raw === "") return 0.8
+    const v = Number(raw)
+    return Number.isNaN(v) ? 0.8 : Math.max(0, Math.min(1.5, v))
+  }
+
+  constructor() {
+    const Ctor = window.AudioContext || (window as any).webkitAudioContext
+    this.ctx = new Ctor()
+    this.analyser = this.ctx.createAnalyser()
+    this.analyser.fftSize = 256
+    this.analyser.smoothingTimeConstant = 0.6
+    this.gain = this.ctx.createGain()
+    this.gain.gain.value = AudioQueue.readVolume()
+    this.analyser.connect(this.gain)
+    this.gain.connect(this.ctx.destination)
+    this.freq = new Uint8Array(this.analyser.frequencyBinCount)
+    window.addEventListener("lucy-volume", (e: Event) => {
+      const v = Number((e as CustomEvent).detail)
+      if (!Number.isNaN(v)) this.gain.gain.value = Math.max(0, Math.min(1.5, v))
+    })
+  }
+
+  async enqueue(buf: ArrayBuffer) {
+    this.queue.push(buf)
+    if (!this.playing) await this.playNext()
+  }
+
+  clear() {
+    this.queue = []
+    // laufenden Stream stoppen (Barge-in)
+    this.streamCancelled = true
+    for (const s of this.streamSources) { try { s.stop() } catch { /* */ } }
+    this.streamSources = []
+  }
+
+  // Spielt einen fortlaufenden PCM16-mono-Stream lueckenlos ab: jede Frame-Charge wird auf der
+  // Audio-Uhr direkt hinter die vorige geplant (kein onended-Gap). LEAD_IN puffert gegen Underruns.
+  async playPcmStream(stream: ReadableStream<Uint8Array>, sampleRate: number): Promise<void> {
+    if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } }
+    this.streamCancelled = false
+    this.streamSources = []
+    const reader = stream.getReader()
+    const LEAD_IN = 0.35   // Startpuffer: mehr Vorlauf -> Generierung bleibt vor der Wiedergabe (weniger Unterläufe/Stocken)
+    let nextTime = 0, started = false, leftoverByte = -1, lastEnd = 0
+    this.onSpeaking?.(true)
+    this.startMeter()
+    try {
+      for (;;) {
+        const { done, value } = await reader.read()
+        if (done || this.streamCancelled) break
+        if (!value || value.length === 0) continue
+        // ungerades Rest-Byte der vorigen Charge voranstellen, damit Int16-Frames sauber bleiben
+        let bytes: Uint8Array
+        if (leftoverByte >= 0) {
+          bytes = new Uint8Array(value.length + 1)
+          bytes[0] = leftoverByte
+          bytes.set(value, 1)
+        } else {
+          bytes = value
+        }
+        const usable = bytes.length - (bytes.length % 2)
+        leftoverByte = usable < bytes.length ? bytes[bytes.length - 1] : -1
+        if (usable === 0) continue
+        // in ein eigenes, 2-Byte-ausgerichtetes Buffer kopieren (value.byteOffset evtl. ungerade)
+        const aligned = new Uint8Array(usable)
+        aligned.set(bytes.subarray(0, usable))
+        const i16 = new Int16Array(aligned.buffer)
+        const f32 = new Float32Array(i16.length)
+        for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 32768
+        const audioBuf = this.ctx.createBuffer(1, f32.length, sampleRate)
+        audioBuf.copyToChannel(f32, 0)
+        const src = this.ctx.createBufferSource()
+        src.buffer = audioBuf
+        src.connect(this.analyser)
+        if (!started) { nextTime = this.ctx.currentTime + LEAD_IN; started = true }
+        if (nextTime < this.ctx.currentTime) nextTime = this.ctx.currentTime + 0.02  // Underrun-Schutz
+        src.start(nextTime)
+        this.streamSources.push(src)
+        src.onended = () => {
+          const i = this.streamSources.indexOf(src)
+          if (i >= 0) this.streamSources.splice(i, 1)
+        }
+        nextTime += audioBuf.duration
+        lastEnd = nextTime
+      }
+    } finally {
+      try { reader.releaseLock() } catch { /* */ }
+    }
+    // bis zum Ende der letzten geplanten Charge warten (sofern nicht abgebrochen)
+    if (!this.streamCancelled) {
+      const waitMs = Math.max(0, (lastEnd - this.ctx.currentTime) * 1000)
+      await new Promise((r) => setTimeout(r, waitMs + 60))
+    }
+    this.onSpeaking?.(false)
+    this.stopMeter()
+  }
+
+  private async playNext(): Promise<void> {
+    const buf = this.queue.shift()
+    if (!buf) { this.playing = false; this.stopMeter(); this.onSpeaking?.(false); return }
+    this.playing = true
+    this.onSpeaking?.(true)
+    if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } }
+    let audioBuf: AudioBuffer
+    try { audioBuf = await this.ctx.decodeAudioData(buf.slice(0)) } catch { return this.playNext() }
+    const src = this.ctx.createBufferSource()
+    src.buffer = audioBuf
+    src.connect(this.analyser)
+    src.onended = () => { void this.playNext() }
+    src.start()
+    this.startMeter()
+  }
+
+  private startMeter() {
+    cancelAnimationFrame(this.raf)
+    const tick = () => {
+      this.analyser.getByteFrequencyData(this.freq as any)
+      const n = Math.min(this.freq.length, 48)
+      let sum = 0, wsum = 0
+      for (let i = 2; i < n; i++) { const m = this.freq[i]; sum += m; wsum += m * i }
+      const avg = sum / (n - 2) / 255
+      const open = Math.min(1, avg * 1.9)
+      this.level.current = open
+      // Spektraler Schwerpunkt 0..1 (wo sitzt die Klang-Energie) -> grobe Vokal-Form
+      const c = sum > 0 ? wsum / sum : 2
+      const cN = Math.max(0, Math.min(1, (c - 2) / (n - 2)))
+      const bright = smoothstep(0.42, 0.72, cN)            // hell -> i/e (breiter Mund)
+      const dark = 1 - smoothstep(0.22, 0.5, cN)           // dunkel -> o/u (runder Mund)
+      const ih = open * bright
+      const ou = open * dark
+      const aa = open * (1 - Math.max(bright, dark) * 0.85) // sonst offenes "a"
+      // leichte Glaettung gegen Flackern
+      this.level.aa += (aa - this.level.aa) * 0.5
+      this.level.ih += (ih - this.level.ih) * 0.5
+      this.level.ou += (ou - this.level.ou) * 0.5
+      this.raf = requestAnimationFrame(tick)
+    }
+    tick()
+  }
+
+  private stopMeter() {
+    cancelAnimationFrame(this.raf)
+    this.level.current = 0; this.level.aa = 0; this.level.ih = 0; this.level.ou = 0
+  }
+}