Hitonabi/mission-control-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lucy v2 (MateEngine-Verhalten): Blickfolgen + Streicheln + Ziehen

Browse Source 
- Mauszeiger-Verfolgung: Main pollt globalen Cursor (screen.getCursorScreenPoint), Renderer richtet
  Augen (lookAt-Offset in Kamera-Achsen) + dezent Kopf/Nacken danach aus
- Antippen/Streicheln: Klick auf den Avatar -> kurze freudige Reaktion (Laecheln + Kopf-Wackeln)
- Ziehen: im Overlay ist der Avatar eine Drag-Region (am Koerper greifen -> Fenster verschieben),
  Kamera-Drehung (OrbitControls) im Overlay aus; Blasen/Knoepfe bleiben no-drag
- neue IPC-Events: lucy:cursor (+ preload onCursor)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
2026-06-30 19:35:06 +02:00
parent feba73a11a
commit c3c7c2ca91
6 changed files with 75 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/lucy-desktop/src/main/index.ts
+19 -2
View File
@@ -1,4 +1,4 @@
import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell } from "electron"
import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen } from "electron"
import { spawn, ChildProcess } from "child_process"
import { join } from "path"
@@ -20,6 +20,22 @@ let tray: Tray | null = null
let ttsProc: ChildProcess | null = null
let mode: "full" | "overlay" = "full"
let pinned = false
let cursorTimer: ReturnType<typeof setInterval> | null = null
// MateEngine-Idee: Lucy folgt dem Mauszeiger. Der Zeiger ist auch AUSSERHALB des Fensters (Geistmodus),
// daher global pollen (screen.getCursorScreenPoint) und die Richtung relativ zu Lucys Kopf an den Renderer geben.
function startCursorTracking() {
if (cursorTimer) return
cursorTimer = setInterval(() => {
if (!win || win.isDestroyed() || !win.isVisible()) return
const b = win.getBounds()
const p = screen.getCursorScreenPoint()
const cx = b.x + b.width / 2
const cy = b.y + b.height * 0.38 // ungefaehre Kopfhoehe im Fenster
win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) })
}, 40) // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last
}
function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } }
async function ensureTtsServer() {
try {
@@ -150,8 +166,9 @@ app.whenReady().then(() => {
createWindow()
buildTray()
registerHotkey()
startCursorTracking()
app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() })
})
app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ })
app.on("before-quit", () => { stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() })
app.on("before-quit", () => { stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() })
client/lucy-desktop/src/preload/index.ts
+6
View File
@@ -25,4 +25,10 @@ contextBridge.exposeInMainWorld("lucy", {
ipcRenderer.on("lucy:hotkey", h)
return () => ipcRenderer.removeListener("lucy:hotkey", h)
},
// Mauszeiger-Position (relativ zu Lucys Kopf) -> Blickfolgen
onCursor: (cb: (p: { x: number; y: number }) => void) => {
const h = (_e: unknown, p: { x: number; y: number }) => cb(p)
ipcRenderer.on("lucy:cursor", h)
return () => ipcRenderer.removeListener("lucy:cursor", h)
},
})
client/lucy-desktop/src/renderer/src/App.tsx
+8 -2
View File
@@ -21,6 +21,8 @@ export default function App() {
inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
const holding = useRef(false)
const endRef = useRef<HTMLDivElement>(null)
const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
const pat = useRef(0) // Zaehler: Antippen/Streicheln des Avatars
const [mode, setMode] = useState<"full" | "overlay">("full")
const [pinned, setPinned] = useState(false)
const [ghost, setGhost] = useState(false) // Geistmodus: Fenster durchklickbar (nur Overlay)
@@ -49,6 +51,9 @@ export default function App() {
}, [ghost, mode])
useEffect(() => { if (mode !== "overlay") setGhost(false) }, [mode]) // Geistmodus nur im Overlay
// Mauszeiger-Verfolgung (MateEngine-Idee): Position vom Main-Prozess in den Avatar-Blick fuettern
useEffect(() => window.lucy?.onCursor((p) => { cursor.current = p }), [])
// Leertaste = Push-to-talk (halten)
useEffect(() => {
const isField = (el: EventTarget | null) =>
@@ -126,10 +131,11 @@ export default function App() {
<div className="body">
<div className="stage">
<div className="avatarWrap">
<div className="avatarWrap" onClick={() => { pat.current += 1 }}
title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
<div className="digitalRoom"><div className="dust" /></div>
<AuraGlow status={status} audioLevel={audioLevel} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} controls={!overlay} />
{speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
{!ready && (
<div className="warmOverlay">
client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx
+38 -6
View File
@@ -14,6 +14,8 @@ import type { Emotion } from "../lib/voice/sentiment"
type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }>
type EmotionRef = MutableRefObject<Emotion>
type CursorRef = MutableRefObject<{ x: number; y: number }>
type NumRef = MutableRefObject<number>
// Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md).
// Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen,
@@ -75,8 +77,9 @@ const EMO_TO_EXPR: Record<Emotion, string | null> = {
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
}
function VrmModel({ url, audioLevel, emotion, status, onError }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>; onError: (m: string) => void
function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void
}) {
const [vrm, setVrm] = useState<VRM | null>(null)
const smooth = useRef<Record<string, number>>({})
@@ -86,6 +89,10 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
const gaze = useRef(new Object3D()) // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera)
const headPos = useRef(new Vector3())
const camPos = useRef(new Vector3())
const camRight = useRef(new Vector3())
const camUp = useRef(new Vector3())
const patReact = useRef(0) // 0..1 Reaktion aufs Antippen/Streicheln
const lastPat = useRef(0)
// Bewegungs-Clips
const mixer = useRef<AnimationMixer | null>(null)
const actions = useRef<Record<string, AnimationAction>>({})
@@ -182,6 +189,17 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
} else {
applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf
}
// MateEngine-Idee: Kopf dreht sich dezent zum Mauszeiger (Augen folgen unten im lookAt-Block)
const cur = cursor?.current
if (cur) {
const hx = Math.max(-1, Math.min(1, cur.x)), hy = Math.max(-1, Math.min(1, cur.y))
addBone(vrm, "neck", hy * 0.06, hx * 0.10, 0)
addBone(vrm, "head", hy * 0.08, hx * 0.14, 0)
}
// Antippen/Streicheln: kurze freudige Reaktion (Kopf-Wackeln; Laecheln in der Mimik unten)
if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
patReact.current *= Math.max(0, 1 - delta * 1.4)
if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
// Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
if (vrm.lookAt) {
@@ -190,6 +208,15 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0)
const cp = camPos.current.copy(state.camera.position)
gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp)
// Augen folgen dem Mauszeiger: Blickpunkt in Kamera-Rechts/Hoch-Richtung verschieben
if (cur) {
const e = state.camera.matrixWorld.elements
camRight.current.set(e[0], e[1], e[2]).normalize()
camUp.current.set(e[4], e[5], e[6]).normalize()
const cx = Math.max(-1.5, Math.min(1.5, cur.x)), cy = Math.max(-1.5, Math.min(1.5, cur.y))
gaze.current.position.addScaledVector(camRight.current, cx * 1.8)
gaze.current.position.addScaledVector(camUp.current, -cy * 1.8)
}
// dezente Blick-Mikrobewegung (Sakkaden) -> das Gesicht wirkt nicht eingefroren
const s = sac.current
s.t += delta
@@ -225,6 +252,8 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
const expr = brk.current.name === "Relax" ? "happy" : "relaxed"
em.setValue(expr, Math.max(smooth.current[expr] ?? 0, brk.current.w * 0.35))
}
// Streichel-Reaktion: Laecheln
if (patReact.current > 0.01) em.setValue("happy", Math.max(smooth.current.happy ?? 0, patReact.current * 0.85))
const b = blink.current
b.t += delta
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
@@ -238,8 +267,9 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: {
return vrm ? <primitive object={vrm.scene} /> : null
}
export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: {
export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
cursor?: CursorRef; pat?: NumRef; controls?: boolean
}) {
const [err, setErr] = useState<string | null>(null)
const statusRef = useRef(status); statusRef.current = status
@@ -249,9 +279,11 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: {
<ambientLight intensity={0.85} />
<directionalLight position={[1, 2, 2]} intensity={1.1} />
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} onError={setErr} />
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} onError={setErr} />
{controls && (
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
)}
</Canvas>
{err && (
<div style={{ position: "absolute", left: 0, right: 0, bottom: 12, margin: "0 auto", width: "fit-content",
client/lucy-desktop/src/renderer/src/config.ts
+1
View File
@@ -28,6 +28,7 @@ declare global {
togglePin: () => Promise<boolean>
onMode: (cb: (m: "full" | "overlay") => void) => () => void
onHotkey: (cb: () => void) => () => void
onCursor: (cb: (p: { x: number; y: number }) => void) => () => void
captureScreen: () => Promise<string[]>
listWindows: () => Promise<{ id: string; name: string }[]>
captureWindow: (id: string) => Promise<string | null>
client/lucy-desktop/src/renderer/src/styles.css
+3
View File
@@ -43,6 +43,9 @@ body {
}
.app.overlay .stage { border: none; background: transparent; }
.avatarWrap { flex: 1; min-height: 0; position: relative; overflow: hidden; border-radius: 14px; }
/* Overlay: Lucy am Koerper greifen & ueber den Desktop ziehen (Bedienelemente bleiben no-drag) */
.app.overlay .avatarWrap { -webkit-app-region: drag; }
.app.overlay .overlayBubbles, .app.overlay .overlayBubbles * { -webkit-app-region: no-drag; }
/* --- digitaler Raum (animierter Sci-Fi-Backdrop, Etappe 2B) --- */
.digitalRoom {