diff --git a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx new file mode 100644 index 0000000..698ec67 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx @@ -0,0 +1,185 @@ +import { Canvas, useFrame } from "@react-three/fiber" +import { OrbitControls } from "@react-three/drei" +import { useEffect, useRef, useState, type MutableRefObject } from "react" +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js" +import { Object3D, Vector3 } from "three" +import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" +import type { Emotion } from "../lib/voice/sentiment" + +// 3D-Avatar (VRM): Lippensync (Mund folgt TTS-Pegel), Blinzeln, Mimik, lebendige Idle-/Sprech-Bewegung. +// Liest Live-Werte aus Mutable-Refs (kein Re-Render pro Frame). + +type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }> +type EmotionRef = MutableRefObject + +const REST: Record = { + leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0], +} +function setBone(vrm: VRM, name: string, x: number, y: number, z: number) { + const b = vrm.humanoid?.getNormalizedBoneNode(name as any) + if (b) b.rotation.set(x, y, z) +} +function applyRestPose(vrm: VRM) { + for (const [name, r] of Object.entries(REST)) setBone(vrm, name, r[0], r[1], r[2]) + vrm.humanoid?.update() +} + +function applyIdle(vrm: VRM, t: number, level: number, lookYaw: number, lookPitch: number, lean: number, speak: number) { + const breathe = Math.sin(t * 1.6), sway = Math.sin(t * 0.45), weight = Math.sin(t * 0.32), weight2 = Math.sin(t * 0.21 + 1) + const emph = Math.min(1, level * 1.6) + // Sprech-Rhythmus: organischer Doppel-Takt -> Kopf nickt/betont auf der Stimme + const beat = Math.sin(t * 2.4) + Math.sin(t * 3.7) * 0.5 + const nod = (emph * 0.05 + speak * 0.02) * beat + emph * Math.sin(t * 1.3) * 0.03 + const tilt = Math.sin(t * 0.7 + 0.5) * 0.03 + speak * Math.sin(t * 0.9) * 0.045 // gelegentliches Kopf-Neigen + setBone(vrm, "hips", 0, weight * 0.06, weight * 0.04 + weight2 * 0.02) + setBone(vrm, "spine", breathe * 0.03 + lean * 0.08 + speak * emph * 0.03, sway * 0.03, -weight * 0.04) + setBone(vrm, "chest", breathe * 0.025 + lean * 0.025, sway * 0.022, weight2 * 0.012) + setBone(vrm, "upperChest", breathe * 0.018, sway * 0.01, 0) + setBone(vrm, "neck", lookPitch * 0.4 + nod * 0.5, lookYaw * 0.4, tilt * 0.5) + setBone(vrm, "head", lookPitch * 0.6 + nod + Math.sin(t * 0.6) * 0.02, + lookYaw * 0.6 + Math.sin(t * 0.27) * 0.035, tilt + Math.sin(t * 0.5) * 0.025) + // Arme: lebhafterer Schwung um die Ruhepose + dezente Sprech-Geste — SICHER (kleine Amplituden) + const armSwing = Math.sin(t * 0.8) * 0.05 + speak * Math.sin(t * 1.6) * 0.045 + const lift = speak * emph * 0.07 // hebt die Arme beim Reden minimal an (mit den Händen reden) + setBone(vrm, "leftUpperArm", -lift, 0, 1.16 + armSwing + weight * 0.05) + setBone(vrm, "rightUpperArm", -lift, 0, -1.16 - armSwing + weight * 0.05) + setBone(vrm, "leftLowerArm", 0, -0.18 - Math.sin(t * 0.8) * 0.04 - speak * 0.07, 0) + setBone(vrm, "rightLowerArm", 0, 0.18 + Math.sin(t * 0.8) * 0.04 + speak * 0.07, 0) +} + +const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const +const EMO_TO_EXPR: Record = { + neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed", +} + +function VrmModel({ url, audioLevel, emotion, status, onError }: { + url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject; onError: (m: string) => void +}) { + const [vrm, setVrm] = useState(null) + const smooth = useRef>({}) + const blink = useRef({ t: 0, next: 3, active: 0 }) + const motion = useRef({ yaw: 0, pitch: 0, tYaw: 0, tPitch: 0, t: 0, next: 2.5, lean: 0 }) + const think = useRef(0) // 0..1 Nachdenk-Pose-Blend + const speak = useRef(0) // 0..1 Sprech-Lebendigkeit + const gaze = useRef(new Object3D()) // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera) + const headPos = useRef(new Vector3()) + const camPos = useRef(new Vector3()) + + useEffect(() => { + let disposed = false, loaded: VRM | null = null + const loader = new GLTFLoader() + loader.register((parser) => new VRMLoaderPlugin(parser)) + loader.load(url, (gltf) => { + if (disposed) return + const v = gltf.userData.vrm as VRM | undefined + if (!v) { onError("Datei enthaelt kein gueltiges VRM-Modell."); return } + VRMUtils.removeUnnecessaryVertices(gltf.scene) + if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v) + v.scene.rotation.y = Math.PI + applyRestPose(v) + loaded = v; setVrm(v) + }, undefined, (err) => { console.error("VRM-Load:", err); onError("Avatar konnte nicht geladen werden.") }) + return () => { disposed = true; if (loaded) VRMUtils.deepDispose(loaded.scene); setVrm(null) } + }, [url, onError]) + + useFrame((state, delta) => { + if (!vrm) return + const target = audioLevel.current?.current ?? 0 + // Blick zur Kamera, aber auf einen WEIT entfernten Punkt in Kamerarichtung -> Augen laufen + // nicht zusammen (kein Schielen bei naher Kamera), wirken aber weiter "auf dich gerichtet". + if (vrm.lookAt) { + const headNode = vrm.humanoid?.getNormalizedBoneNode("head") + const hp = headPos.current + if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0) + const cp = camPos.current.copy(state.camera.position) + gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp) // 4x = Blickpunkt weit dahinter + vrm.lookAt.target = gaze.current + } + const st = status.current + const spk = (speak.current += (((st === "speaking") ? 1 : 0) - speak.current) * Math.min(1, delta * 4)) + const m = motion.current + m.t += delta + if (m.t > m.next) { + // beim Sprechen häufiger + eher zur Kamera (engagiert), sonst lebhaftes Umschauen + const range = spk > 0.5 ? 0.28 : 0.6 + m.tYaw = (Math.random() - 0.5) * range; m.tPitch = (Math.random() - 0.5) * 0.3 + m.t = 0; m.next = (spk > 0.5 ? 1.4 : 1.9) + Math.random() * 2.6 + } + m.yaw += (m.tYaw - m.yaw) * Math.min(1, delta * 1.8) + m.pitch += (m.tPitch - m.pitch) * Math.min(1, delta * 1.8) + // Nachdenk-Blend hoch wenn 'thinking'/'transcribing', sonst runter + const wantThink = (st === "thinking" || st === "transcribing") ? 1 : 0 + think.current += (wantThink - think.current) * Math.min(1, delta * 2.5) + const th = think.current + const attentive = st === "listening" ? 1 : 0 + m.lean += ((Math.min(1, target * 1.6) + attentive * 0.5) - m.lean) * Math.min(1, delta * 3) + // Beim Nachdenken: Blick zur Seite/oben (pensiv, weg von der Kamera) statt zufälligem Umschauen + const tc = state.clock.elapsedTime + // Nachdenken: Blick zur Seite/leicht nach oben (pensiv, weg von der Kamera) + Kopf-Neigung. + // KEINE Arm-Bewegung (risikolos; die Hand-zum-Kinn-Pose liess sich blind nicht sauber treffen). + const yaw = m.yaw + th * (-0.38 + Math.sin(tc * 0.5) * 0.05) + const pitch = m.pitch + th * (-0.12) + applyIdle(vrm, tc, target * (1 - th * 0.5), yaw, pitch, m.lean, spk) + if (th > 0.02) { + // sanftes nachdenkliches Kopf-Neigen (nur Roll dazu, Idle-Kopfbewegung bleibt erhalten) + setBone(vrm, "head", pitch * 0.6 + Math.sin(tc * 0.6) * 0.015, yaw * 0.6 + Math.sin(tc * 0.27) * 0.025, + Math.sin(tc * 0.5) * 0.02 + th * 0.14) + vrm.humanoid?.update() + } + + const em = vrm.expressionManager + if (em) { + // Lippensync v2: echte Vokal-Mundformen (aa/ih/ou) aus der Audio-Analyse statt nur "Mund auf". + // Fallback auf den Gesamtpegel (target) fuer aa, falls keine Mundform-Werte vorliegen. + const lv = audioLevel.current + const visTargets: Record = { + aa: lv?.aa ?? target, ih: lv?.ih ?? 0, ou: lv?.ou ?? 0, + } + for (const v of ["aa", "ih", "ou"]) { + const cv = smooth.current[v] ?? 0 + const nv = cv + (visTargets[v] - cv) * Math.min(1, delta * 14) // schnell genug fuers Sprechen + smooth.current[v] = nv; em.setValue(v, nv) + } + const want = EMO_TO_EXPR[emotion.current] + for (const name of EXPRESSIONS) { + const tv = want === name ? 0.75 : 0 + const cv = smooth.current[name] ?? 0 + const nv = cv + (tv - cv) * Math.min(1, delta * 4) + smooth.current[name] = nv; em.setValue(name, nv) + } + const b = blink.current + b.t += delta + if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 } + let blinkVal = 0 + if (b.active > 0) { b.active -= delta; const p = 1 - b.active / 0.16; blinkVal = 1 - Math.abs(p - 0.5) * 2 } + em.setValue("blink", Math.max(0, blinkVal)) + } + vrm.update(delta) + }) + + return vrm ? : null +} + +export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: { + url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string +}) { + const [err, setErr] = useState(null) + const statusRef = useRef(status); statusRef.current = status + return ( +
+ + + + + + + + {err && ( +
{err}
+ )} +
+ ) +} diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/audio.ts b/client/lucy-desktop/src/renderer/src/lib/voice/audio.ts new file mode 100644 index 0000000..f7a3921 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/lib/voice/audio.ts @@ -0,0 +1,175 @@ +// Audio-Wiedergabe + Pegelmessung fuers Lippensync (Avatar liest level.current). +// Zwei Wege: enqueue(WAV-Buffer) [z.B. Warmup] und playPcmStream(PCM16-Stream) [die Live-Antwort, +// lueckenlos via Web-Audio-Scheduling, niedrige Time-to-first-audio]. +// +// LIPPENSYNC v2: statt nur einem Pegel ("Mund auf/zu") leiten wir aus dem Frequenz-Spektrum +// echte Vokal-MUNDFORMEN ab (level.aa/ih/ou). Idee: die Helligkeit des Klangs (spektraler +// Schwerpunkt) verraet grob den Vokal — helle Laute (i/e) = breiter Mund, dunkle (o/u) = runder +// Mund, mittig = offenes "a". Das ist nicht phonetisch exakt, sieht aber lebendig+passend aus. +function smoothstep(a: number, b: number, x: number): number { + const t = Math.max(0, Math.min(1, (x - a) / (b - a))) + return t * t * (3 - 2 * t) +} + +export class AudioQueue { + private ctx: AudioContext + private analyser: AnalyserNode + private gain: GainNode + private queue: ArrayBuffer[] = [] + private playing = false + private raf = 0 + private freq: Uint8Array + // aktiver PCM-Stream (fuer Barge-in/clear): geplante Quellen + Abbruchsignal + private streamSources: AudioBufferSourceNode[] = [] + private streamCancelled = false + // current = Gesamt-Pegel (Aura/Bewegung); aa/ih/ou = Mundform-Gewichte fuer den Avatar + readonly level = { current: 0, aa: 0, ih: 0, ou: 0 } + onSpeaking?: (speaking: boolean) => void + + static readVolume(): number { + const raw = localStorage.getItem("lucy_volume") + if (raw === null || raw === "") return 0.8 + const v = Number(raw) + return Number.isNaN(v) ? 0.8 : Math.max(0, Math.min(1.5, v)) + } + + constructor() { + const Ctor = window.AudioContext || (window as any).webkitAudioContext + this.ctx = new Ctor() + this.analyser = this.ctx.createAnalyser() + this.analyser.fftSize = 256 + this.analyser.smoothingTimeConstant = 0.6 + this.gain = this.ctx.createGain() + this.gain.gain.value = AudioQueue.readVolume() + this.analyser.connect(this.gain) + this.gain.connect(this.ctx.destination) + this.freq = new Uint8Array(this.analyser.frequencyBinCount) + window.addEventListener("lucy-volume", (e: Event) => { + const v = Number((e as CustomEvent).detail) + if (!Number.isNaN(v)) this.gain.gain.value = Math.max(0, Math.min(1.5, v)) + }) + } + + async enqueue(buf: ArrayBuffer) { + this.queue.push(buf) + if (!this.playing) await this.playNext() + } + + clear() { + this.queue = [] + // laufenden Stream stoppen (Barge-in) + this.streamCancelled = true + for (const s of this.streamSources) { try { s.stop() } catch { /* */ } } + this.streamSources = [] + } + + // Spielt einen fortlaufenden PCM16-mono-Stream lueckenlos ab: jede Frame-Charge wird auf der + // Audio-Uhr direkt hinter die vorige geplant (kein onended-Gap). LEAD_IN puffert gegen Underruns. + async playPcmStream(stream: ReadableStream, sampleRate: number): Promise { + if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } } + this.streamCancelled = false + this.streamSources = [] + const reader = stream.getReader() + const LEAD_IN = 0.35 // Startpuffer: mehr Vorlauf -> Generierung bleibt vor der Wiedergabe (weniger Unterläufe/Stocken) + let nextTime = 0, started = false, leftoverByte = -1, lastEnd = 0 + this.onSpeaking?.(true) + this.startMeter() + try { + for (;;) { + const { done, value } = await reader.read() + if (done || this.streamCancelled) break + if (!value || value.length === 0) continue + // ungerades Rest-Byte der vorigen Charge voranstellen, damit Int16-Frames sauber bleiben + let bytes: Uint8Array + if (leftoverByte >= 0) { + bytes = new Uint8Array(value.length + 1) + bytes[0] = leftoverByte + bytes.set(value, 1) + } else { + bytes = value + } + const usable = bytes.length - (bytes.length % 2) + leftoverByte = usable < bytes.length ? bytes[bytes.length - 1] : -1 + if (usable === 0) continue + // in ein eigenes, 2-Byte-ausgerichtetes Buffer kopieren (value.byteOffset evtl. ungerade) + const aligned = new Uint8Array(usable) + aligned.set(bytes.subarray(0, usable)) + const i16 = new Int16Array(aligned.buffer) + const f32 = new Float32Array(i16.length) + for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 32768 + const audioBuf = this.ctx.createBuffer(1, f32.length, sampleRate) + audioBuf.copyToChannel(f32, 0) + const src = this.ctx.createBufferSource() + src.buffer = audioBuf + src.connect(this.analyser) + if (!started) { nextTime = this.ctx.currentTime + LEAD_IN; started = true } + if (nextTime < this.ctx.currentTime) nextTime = this.ctx.currentTime + 0.02 // Underrun-Schutz + src.start(nextTime) + this.streamSources.push(src) + src.onended = () => { + const i = this.streamSources.indexOf(src) + if (i >= 0) this.streamSources.splice(i, 1) + } + nextTime += audioBuf.duration + lastEnd = nextTime + } + } finally { + try { reader.releaseLock() } catch { /* */ } + } + // bis zum Ende der letzten geplanten Charge warten (sofern nicht abgebrochen) + if (!this.streamCancelled) { + const waitMs = Math.max(0, (lastEnd - this.ctx.currentTime) * 1000) + await new Promise((r) => setTimeout(r, waitMs + 60)) + } + this.onSpeaking?.(false) + this.stopMeter() + } + + private async playNext(): Promise { + const buf = this.queue.shift() + if (!buf) { this.playing = false; this.stopMeter(); this.onSpeaking?.(false); return } + this.playing = true + this.onSpeaking?.(true) + if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } } + let audioBuf: AudioBuffer + try { audioBuf = await this.ctx.decodeAudioData(buf.slice(0)) } catch { return this.playNext() } + const src = this.ctx.createBufferSource() + src.buffer = audioBuf + src.connect(this.analyser) + src.onended = () => { void this.playNext() } + src.start() + this.startMeter() + } + + private startMeter() { + cancelAnimationFrame(this.raf) + const tick = () => { + this.analyser.getByteFrequencyData(this.freq as any) + const n = Math.min(this.freq.length, 48) + let sum = 0, wsum = 0 + for (let i = 2; i < n; i++) { const m = this.freq[i]; sum += m; wsum += m * i } + const avg = sum / (n - 2) / 255 + const open = Math.min(1, avg * 1.9) + this.level.current = open + // Spektraler Schwerpunkt 0..1 (wo sitzt die Klang-Energie) -> grobe Vokal-Form + const c = sum > 0 ? wsum / sum : 2 + const cN = Math.max(0, Math.min(1, (c - 2) / (n - 2))) + const bright = smoothstep(0.42, 0.72, cN) // hell -> i/e (breiter Mund) + const dark = 1 - smoothstep(0.22, 0.5, cN) // dunkel -> o/u (runder Mund) + const ih = open * bright + const ou = open * dark + const aa = open * (1 - Math.max(bright, dark) * 0.85) // sonst offenes "a" + // leichte Glaettung gegen Flackern + this.level.aa += (aa - this.level.aa) * 0.5 + this.level.ih += (ih - this.level.ih) * 0.5 + this.level.ou += (ou - this.level.ou) * 0.5 + this.raf = requestAnimationFrame(tick) + } + tick() + } + + private stopMeter() { + cancelAnimationFrame(this.raf) + this.level.current = 0; this.level.aa = 0; this.level.ih = 0; this.level.ou = 0 + } +}