Hitonabi/mission-control-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lucy v2 (MateEngine-Verhalten): Blickfolgen + Streicheln + Ziehen

Browse Source 
- Mauszeiger-Verfolgung: Main pollt globalen Cursor (screen.getCursorScreenPoint), Renderer richtet
  Augen (lookAt-Offset in Kamera-Achsen) + dezent Kopf/Nacken danach aus
- Antippen/Streicheln: Klick auf den Avatar -> kurze freudige Reaktion (Laecheln + Kopf-Wackeln)
- Ziehen: im Overlay ist der Avatar eine Drag-Region (am Koerper greifen -> Fenster verschieben),
  Kamera-Drehung (OrbitControls) im Overlay aus; Blasen/Knoepfe bleiben no-drag
- neue IPC-Events: lucy:cursor (+ preload onCursor)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
2026-06-30 19:35:06 +02:00
parent feba73a11a
commit c3c7c2ca91
6 changed files with 75 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/lucy-desktop/src/renderer/src/App.tsx
+8 -2
View File
@@ -21,6 +21,8 @@ export default function App() {
inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
const holding = useRef(false)
const endRef = useRef<HTMLDivElement>(null)
const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
const pat = useRef(0) // Zaehler: Antippen/Streicheln des Avatars
const [mode, setMode] = useState<"full" | "overlay">("full")
const [pinned, setPinned] = useState(false)
const [ghost, setGhost] = useState(false) // Geistmodus: Fenster durchklickbar (nur Overlay)
@@ -49,6 +51,9 @@ export default function App() {
}, [ghost, mode])
useEffect(() => { if (mode !== "overlay") setGhost(false) }, [mode]) // Geistmodus nur im Overlay
// Mauszeiger-Verfolgung (MateEngine-Idee): Position vom Main-Prozess in den Avatar-Blick fuettern
useEffect(() => window.lucy?.onCursor((p) => { cursor.current = p }), [])
// Leertaste = Push-to-talk (halten)
useEffect(() => {
const isField = (el: EventTarget | null) =>
@@ -126,10 +131,11 @@ export default function App() {
<div className="body">
<div className="stage">
<div className="avatarWrap">
<div className="avatarWrap" onClick={() => { pat.current += 1 }}
title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
<div className="digitalRoom"><div className="dust" /></div>
<AuraGlow status={status} audioLevel={audioLevel} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} controls={!overlay} />
{speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
{!ready && (
<div className="warmOverlay">
client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx
+38 -6
View File
@@ -14,6 +14,8 @@ import type { Emotion } from "../lib/voice/sentiment"
type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }>
type EmotionRef = MutableRefObject<Emotion>
type CursorRef = MutableRefObject<{ x: number; y: number }>
type NumRef = MutableRefObject<number>
// Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md).
// Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen,
@@ -75,8 +77,9 @@ const EMO_TO_EXPR: Record<Emotion, string | null> = {
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
}
function VrmModel({ url, audioLevel, emotion, status, onError }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>; onError: (m: string) => void
function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void
}) {
const [vrm, setVrm] = useState<VRM | null>(null)
const smooth = useRef<Record<string, number>>({})
@@ -86,6 +89,10 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
const gaze = useRef(new Object3D()) // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera)
const headPos = useRef(new Vector3())
const camPos = useRef(new Vector3())
const camRight = useRef(new Vector3())
const camUp = useRef(new Vector3())
const patReact = useRef(0) // 0..1 Reaktion aufs Antippen/Streicheln
const lastPat = useRef(0)
// Bewegungs-Clips
const mixer = useRef<AnimationMixer | null>(null)
const actions = useRef<Record<string, AnimationAction>>({})
@@ -182,6 +189,17 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
} else {
applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf
}
// MateEngine-Idee: Kopf dreht sich dezent zum Mauszeiger (Augen folgen unten im lookAt-Block)
const cur = cursor?.current
if (cur) {
const hx = Math.max(-1, Math.min(1, cur.x)), hy = Math.max(-1, Math.min(1, cur.y))
addBone(vrm, "neck", hy * 0.06, hx * 0.10, 0)
addBone(vrm, "head", hy * 0.08, hx * 0.14, 0)
}
// Antippen/Streicheln: kurze freudige Reaktion (Kopf-Wackeln; Laecheln in der Mimik unten)
if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
patReact.current *= Math.max(0, 1 - delta * 1.4)
if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
// Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
if (vrm.lookAt) {
@@ -190,6 +208,15 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0)
const cp = camPos.current.copy(state.camera.position)
gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp)
// Augen folgen dem Mauszeiger: Blickpunkt in Kamera-Rechts/Hoch-Richtung verschieben
if (cur) {
const e = state.camera.matrixWorld.elements
camRight.current.set(e[0], e[1], e[2]).normalize()
camUp.current.set(e[4], e[5], e[6]).normalize()
const cx = Math.max(-1.5, Math.min(1.5, cur.x)), cy = Math.max(-1.5, Math.min(1.5, cur.y))
gaze.current.position.addScaledVector(camRight.current, cx * 1.8)
gaze.current.position.addScaledVector(camUp.current, -cy * 1.8)
}
// dezente Blick-Mikrobewegung (Sakkaden) -> das Gesicht wirkt nicht eingefroren
const s = sac.current
s.t += delta
@@ -225,6 +252,8 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
const expr = brk.current.name === "Relax" ? "happy" : "relaxed"
em.setValue(expr, Math.max(smooth.current[expr] ?? 0, brk.current.w * 0.35))
}
// Streichel-Reaktion: Laecheln
if (patReact.current > 0.01) em.setValue("happy", Math.max(smooth.current.happy ?? 0, patReact.current * 0.85))
const b = blink.current
b.t += delta
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
@@ -238,8 +267,9 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
return vrm ? <primitive object={vrm.scene} /> : null
}
export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: {
export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
cursor?: CursorRef; pat?: NumRef; controls?: boolean
}) {
const [err, setErr] = useState<string | null>(null)
const statusRef = useRef(status); statusRef.current = status
@@ -249,9 +279,11 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: {
<ambientLight intensity={0.85} />
<directionalLight position={[1, 2, 2]} intensity={1.1} />
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} onError={setErr} />
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} onError={setErr} />
{controls && (
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
)}
</Canvas>
{err && (
<div style={{ position: "absolute", left: 0, right: 0, bottom: 12, margin: "0 auto", width: "fit-content",
client/lucy-desktop/src/renderer/src/config.ts
+1
View File
@@ -28,6 +28,7 @@ declare global {
togglePin: () => Promise<boolean>
onMode: (cb: (m: "full" | "overlay") => void) => () => void
onHotkey: (cb: () => void) => () => void
onCursor: (cb: (p: { x: number; y: number }) => void) => () => void
captureScreen: () => Promise<string[]>
listWindows: () => Promise<{ id: string; name: string }[]>
captureWindow: (id: string) => Promise<string | null>
client/lucy-desktop/src/renderer/src/styles.css
+3
View File
@@ -43,6 +43,9 @@ body {
}
.app.overlay .stage { border: none; background: transparent; }
.avatarWrap { flex: 1; min-height: 0; position: relative; overflow: hidden; border-radius: 14px; }
/* Overlay: Lucy am Koerper greifen & ueber den Desktop ziehen (Bedienelemente bleiben no-drag) */
.app.overlay .avatarWrap { -webkit-app-region: drag; }
.app.overlay .overlayBubbles, .app.overlay .overlayBubbles * { -webkit-app-region: no-drag; }
/* --- digitaler Raum (animierter Sci-Fi-Backdrop, Etappe 2B) --- */
.digitalRoom {