Lucy v2 (Etappe 1.3): gelegentliche Bewegungs-Einschuebe im Leerlauf
- alle ~20-35s ein Einmal-Clip (Strecken/Umschauen), dann zurueck in die Ruhe - bricht sauber ab, sobald zugehoert/gedacht/gesprochen wird - kurze passende Mimik (Laecheln beim Strecken) - (visuelle Abnahme durch User steht noch aus) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,7 @@ import { Canvas, useFrame } from "@react-three/fiber"
|
|||||||
import { OrbitControls } from "@react-three/drei"
|
import { OrbitControls } from "@react-three/drei"
|
||||||
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
||||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
||||||
import { Object3D, Vector3, AnimationMixer, LoopPingPong, type AnimationAction } from "three"
|
import { Object3D, Vector3, AnimationMixer, LoopPingPong, LoopOnce, type AnimationAction } from "three"
|
||||||
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
||||||
import { VRMAnimationLoaderPlugin, createVRMAnimationClip, type VRMAnimation } from "@pixiv/three-vrm-animation"
|
import { VRMAnimationLoaderPlugin, createVRMAnimationClip, type VRMAnimation } from "@pixiv/three-vrm-animation"
|
||||||
import type { Emotion } from "../lib/voice/sentiment"
|
import type { Emotion } from "../lib/voice/sentiment"
|
||||||
@@ -18,7 +18,8 @@ type EmotionRef = MutableRefObject<Emotion>
|
|||||||
// Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md).
|
// Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md).
|
||||||
// Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen,
|
// Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen,
|
||||||
// nicht als Dauerschleife. Ruhiger Leerlauf laeuft prozedural. Weitere Clips liegen auf Platte bereit.
|
// nicht als Dauerschleife. Ruhiger Leerlauf laeuft prozedural. Weitere Clips liegen auf Platte bereit.
|
||||||
const CLIPS = ["Thinking"] as const
|
const CLIPS = ["Thinking", "Relax", "LookAround"] as const
|
||||||
|
const BREAK_CLIPS = ["Relax", "LookAround"] as const // gelegentliche Einmal-Einschuebe im Leerlauf
|
||||||
|
|
||||||
const REST: Record<string, [number, number, number]> = {
|
const REST: Record<string, [number, number, number]> = {
|
||||||
leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0],
|
leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0],
|
||||||
@@ -90,6 +91,7 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
|
|||||||
const actions = useRef<Record<string, AnimationAction>>({})
|
const actions = useRef<Record<string, AnimationAction>>({})
|
||||||
const thinkW = useRef(0) // 0..1 Einblendung der Thinking-Pose
|
const thinkW = useRef(0) // 0..1 Einblendung der Thinking-Pose
|
||||||
const sac = useRef({ x: 0, y: 0, tx: 0, ty: 0, t: 0, next: 1.5 }) // Blick-Mikrobewegung (Sakkaden)
|
const sac = useRef({ x: 0, y: 0, tx: 0, ty: 0, t: 0, next: 1.5 }) // Blick-Mikrobewegung (Sakkaden)
|
||||||
|
const brk = useRef({ name: "", w: 0, t: 0, next: 15, active: false }) // gelegentlicher Bewegungs-Einschub
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let disposed = false, loaded: VRM | null = null
|
let disposed = false, loaded: VRM | null = null
|
||||||
@@ -117,11 +119,11 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
|
|||||||
if (!anims?.[0]) return
|
if (!anims?.[0]) return
|
||||||
const clip = createVRMAnimationClip(anims[0], v)
|
const clip = createVRMAnimationClip(anims[0], v)
|
||||||
const action = mx.clipAction(clip)
|
const action = mx.clipAction(clip)
|
||||||
action.setLoop(LoopPingPong, Infinity) // vor-zurueck -> kein harter Loop-Sprung
|
|
||||||
action.timeScale = 0.85 // etwas ruhiger
|
action.timeScale = 0.85 // etwas ruhiger
|
||||||
action.enabled = true
|
action.enabled = true
|
||||||
action.setEffectiveWeight(0) // startet aus; blendet nur bei Bedarf ein
|
action.setEffectiveWeight(0) // startet aus; blendet nur bei Bedarf ein
|
||||||
action.play()
|
if (name === "Thinking") { action.setLoop(LoopPingPong, Infinity); action.play() } // laeuft immer (Gewicht 0)
|
||||||
|
else { action.setLoop(LoopOnce, 1); action.clampWhenFinished = true } // Einschub: einmal, erst bei Trigger
|
||||||
actions.current[name] = action
|
actions.current[name] = action
|
||||||
}, undefined, (e) => console.warn(`VRMA ${name} nicht geladen:`, e))
|
}, undefined, (e) => console.warn(`VRMA ${name} nicht geladen:`, e))
|
||||||
}
|
}
|
||||||
@@ -143,16 +145,40 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
|
|||||||
const attentive = st === "listening" ? 1 : 0
|
const attentive = st === "listening" ? 1 : 0
|
||||||
lean.current += ((Math.min(1, target * 1.6) + attentive * 0.5) - lean.current) * Math.min(1, delta * 3)
|
lean.current += ((Math.min(1, target * 1.6) + attentive * 0.5) - lean.current) * Math.min(1, delta * 3)
|
||||||
|
|
||||||
// Bewegung: ruhiger PROZEDURALER Leerlauf als Basis (Atmen/leichtes Wiegen); fuers Nachdenken
|
// Bewegung: ruhiger PROZEDURALER Leerlauf als Basis. Echte Clips blenden zustandsabhaengig ein:
|
||||||
// blendet die echte Thinking-Pose ein. Die .vrma sind Gesten -> taugen nicht als Dauerschleife;
|
// Thinking beim Nachdenken (Dauer) + gelegentliche Einmal-Einschuebe im Leerlauf (Strecken/Umschauen
|
||||||
// echte Bewegungs-Einschuebe (Strecken/Umschauen) kommen spaeter als gelegentliche Einmal-Gesten.
|
// alle ~20-35s, dann zurueck in die Ruhe). Die .vrma sind Gesten -> nie als Dauerschleife.
|
||||||
const mx = mixer.current
|
const mx = mixer.current
|
||||||
const thinkAction = mx ? actions.current["Thinking"] : undefined
|
const thinkAction = mx ? actions.current["Thinking"] : undefined
|
||||||
|
const calm = st === "idle"
|
||||||
const wantThink = (st === "thinking" || st === "transcribing") ? 1 : 0
|
const wantThink = (st === "thinking" || st === "transcribing") ? 1 : 0
|
||||||
|
const br = brk.current
|
||||||
|
// Einschub planen: nur wenn ruhig + keiner laeuft; Timer zuruecksetzen, sobald nicht mehr ruhig
|
||||||
|
if (mx && !br.active) {
|
||||||
|
if (calm) {
|
||||||
|
br.t += delta
|
||||||
|
if (br.t > br.next) {
|
||||||
|
const pick = BREAK_CLIPS[Math.floor(Math.random() * BREAK_CLIPS.length)]
|
||||||
|
const a = actions.current[pick]
|
||||||
|
if (a) { a.reset(); a.play(); br.name = pick; br.active = true; br.w = 0 }
|
||||||
|
br.t = 0; br.next = 20 + Math.random() * 15
|
||||||
|
}
|
||||||
|
} else { br.t = 0 }
|
||||||
|
}
|
||||||
thinkW.current += (wantThink - thinkW.current) * Math.min(1, delta * 4)
|
thinkW.current += (wantThink - thinkW.current) * Math.min(1, delta * 4)
|
||||||
if (mx && thinkAction) { thinkAction.setEffectiveWeight(thinkW.current); mx.update(delta) }
|
thinkAction?.setEffectiveWeight(thinkW.current)
|
||||||
if (thinkAction && thinkW.current > 0.05) {
|
// Einschub-Gewicht: ausblenden, wenn nicht mehr ruhig / Nachdenken startet / Clip fast zu Ende
|
||||||
addLife(vrm, tc, spk, lean.current) // Thinking-Clip + Atmen/Nicken obendrauf
|
if (br.active) {
|
||||||
|
const a = actions.current[br.name]
|
||||||
|
const dur = a?.getClip().duration ?? 1
|
||||||
|
const finishing = !calm || wantThink === 1 || (a ? a.time >= dur - 0.4 : true)
|
||||||
|
br.w += ((finishing ? 0 : 1) - br.w) * Math.min(1, delta * 3)
|
||||||
|
a?.setEffectiveWeight(br.w)
|
||||||
|
if (finishing && br.w < 0.02) { a?.stop(); br.active = false; br.name = "" }
|
||||||
|
}
|
||||||
|
if (mx) mx.update(delta)
|
||||||
|
if (thinkW.current > 0.05 || (br.active && br.w > 0.05)) {
|
||||||
|
addLife(vrm, tc, spk, lean.current) // Clip-Pose + Atmen/Nicken obendrauf
|
||||||
} else {
|
} else {
|
||||||
applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf
|
applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf
|
||||||
}
|
}
|
||||||
@@ -194,6 +220,11 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
|
|||||||
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
||||||
smooth.current[name] = nv; em.setValue(name, nv)
|
smooth.current[name] = nv; em.setValue(name, nv)
|
||||||
}
|
}
|
||||||
|
// kurze passende Mimik waehrend eines Bewegungs-Einschubs (Laecheln beim Strecken)
|
||||||
|
if (brk.current.active) {
|
||||||
|
const expr = brk.current.name === "Relax" ? "happy" : "relaxed"
|
||||||
|
em.setValue(expr, Math.max(smooth.current[expr] ?? 0, brk.current.w * 0.35))
|
||||||
|
}
|
||||||
const b = blink.current
|
const b = blink.current
|
||||||
b.t += delta
|
b.t += delta
|
||||||
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
||||||
|
|||||||
Reference in New Issue
Block a user