import { Canvas, useFrame } from "@react-three/fiber" import { OrbitControls } from "@react-three/drei" import { useEffect, useRef, useState, type MutableRefObject } from "react" import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js" import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" import type { Emotion } from "@/lib/voice/sentiment" // 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und // stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame). type LevelRef = MutableRefObject<{ current: number }> // audioLevel.current.current = Pegel 0..1 type EmotionRef = MutableRefObject // Ruhepose (A-Pose) als Basis — VRMs laden sonst in T-Pose (Arme waagerecht). const REST: Record = { leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0], } function setBone(vrm: VRM, name: string, x: number, y: number, z: number) { const b = vrm.humanoid?.getNormalizedBoneNode(name as any) if (b) b.rotation.set(x, y, z) } function applyRestPose(vrm: VRM) { for (const [name, r] of Object.entries(REST)) setBone(vrm, name, r[0], r[1], r[2]) vrm.humanoid?.update() } // Lebendige Idle-Animation: prozedural (kein Animations-File). Über die Ruhepose gelegt: // Atmung, langsames Wiegen + Gewichtsverlagerung, Umschauen (lookYaw/Pitch), Vorlehnen beim // Sprechen (lean) und Arm-Mitbewegung. Die Augen (lookAt) hält die useFrame separat auf die Kamera. function applyIdle(vrm: VRM, t: number, level: number, lookYaw: number, lookPitch: number, lean: number) { const breathe = Math.sin(t * 1.7) // ~0.27 Hz Atmung const sway = Math.sin(t * 0.45) // sanftes Wiegen const weight = Math.sin(t * 0.32) // langsame Gewichtsverlagerung const emphasis = Math.min(1, level * 1.4) const nod = Math.sin(t * 1.3) * emphasis * 0.06 // Sprech-Nicken // Rumpf — Atmung, Wiegen, Gewichtsverlagerung, Vorlehnen beim Sprechen setBone(vrm, "hips", 0, weight * 0.045, weight * 0.03) setBone(vrm, "spine", breathe * 0.025 + lean * 0.07, sway * 0.022, -weight * 0.03) setBone(vrm, "chest", breathe * 0.02 + lean * 0.02, sway * 0.018, 0) setBone(vrm, "upperChest", breathe * 0.015, 0, 0) // Kopf/Hals — Umschauen (gedriftete Zielwinkel) + Atmung + Sprech-Nicken setBone(vrm, "neck", lookPitch * 0.4 + nod * 0.5, lookYaw * 0.4, 0) setBone(vrm, "head", lookPitch * 0.6 + nod * 0.5 + Math.sin(t * 0.6) * 0.015, lookYaw * 0.6 + Math.sin(t * 0.27) * 0.025, Math.sin(t * 0.5) * 0.02) // Arme — Ruhepose + leichtes Pendeln + Gewichtsverlagerung const arm = Math.sin(t * 0.8) * 0.035 setBone(vrm, "leftUpperArm", 0, 0, 1.18 + arm + weight * 0.04) setBone(vrm, "rightUpperArm", 0, 0, -1.18 - arm + weight * 0.04) setBone(vrm, "leftLowerArm", 0, -0.18 - Math.sin(t * 0.8) * 0.03, 0) setBone(vrm, "rightLowerArm", 0, 0.18 + Math.sin(t * 0.8) * 0.03, 0) } const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const const EMO_TO_EXPR: Record = { neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed", } function VrmModel({ url, audioLevel, emotion, onError }: { url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void }) { const [vrm, setVrm] = useState(null) const smooth = useRef>({}) const blink = useRef({ t: 0, next: 3, active: 0 }) // Umschauen (gedriftete Kopf-Zielwinkel) + geglättetes Lehnen beim Sprechen. const motion = useRef({ yaw: 0, pitch: 0, tYaw: 0, tPitch: 0, t: 0, next: 2.5, lean: 0 }) useEffect(() => { let disposed = false let loaded: VRM | null = null const loader = new GLTFLoader() loader.register((parser) => new VRMLoaderPlugin(parser)) loader.load( url, (gltf) => { if (disposed) return const v = gltf.userData.vrm as VRM | undefined if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return } VRMUtils.removeUnnecessaryVertices(gltf.scene) if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v) v.scene.rotation.y = Math.PI // dem Betrachter zuwenden applyRestPose(v) // T-Pose → entspannte A-Pose (Arme unten) loaded = v setVrm(v) }, undefined, (err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") }, ) return () => { disposed = true if (loaded) VRMUtils.deepDispose(loaded.scene) setVrm(null) } }, [url, onError]) useFrame((state, delta) => { if (!vrm) return const target = audioLevel.current?.current ?? 0 // Blickkontakt: Augen folgen der Kamera (schaut dich an). if (vrm.lookAt) vrm.lookAt.target = state.camera // Umschauen: alle paar Sekunden neues Kopf-Ziel, sanft hineinlerpen. const m = motion.current m.t += delta if (m.t > m.next) { m.tYaw = (Math.random() - 0.5) * 0.5 // ±0.25 rad Gieren m.tPitch = (Math.random() - 0.5) * 0.24 m.t = 0 m.next = 2.5 + Math.random() * 3.5 } m.yaw += (m.tYaw - m.yaw) * Math.min(1, delta * 1.5) m.pitch += (m.tPitch - m.pitch) * Math.min(1, delta * 1.5) m.lean += (Math.min(1, target * 1.6) - m.lean) * Math.min(1, delta * 3) // Lebendige Idle-/Sprech-Bewegung (über die Ruhepose gelegt). applyIdle(vrm, state.clock.elapsedTime, target, m.yaw, m.pitch, m.lean) const em = vrm.expressionManager if (em) { // Lippensync: 'aa' folgt geglättet dem Audiopegel. const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6 smooth.current.aa = aa em.setValue("aa", aa) // Mimik: weich zur Ziel-Expression lerpen. const want = EMO_TO_EXPR[emotion.current] for (const name of EXPRESSIONS) { const tv = want === name ? 0.75 : 0 const cv = smooth.current[name] ?? 0 const nv = cv + (tv - cv) * Math.min(1, delta * 4) smooth.current[name] = nv em.setValue(name, nv) } // Blinzeln: kurzer Dreieckspuls alle 3–7 s. const b = blink.current b.t += delta if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 } let blinkVal = 0 if (b.active > 0) { b.active -= delta const p = 1 - b.active / 0.16 // 0..1 Fortschritt blinkVal = 1 - Math.abs(p - 0.5) * 2 // 0 → 1 → 0 } em.setValue("blink", Math.max(0, blinkVal)) } vrm.update(delta) }) return vrm ? : null } export function Avatar3D({ url, audioLevel, emotion }: { url: string; audioLevel: LevelRef; emotion: EmotionRef }) { const [err, setErr] = useState(null) return (
{/* key=url → bei Avatarwechsel sauber neu mounten */} {err && (
{err}
)}
) }