From c3c7c2ca910da7998938be385d26d99cd5ee2efe Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Tue, 30 Jun 2026 19:35:06 +0200 Subject: [PATCH] Lucy v2 (MateEngine-Verhalten): Blickfolgen + Streicheln + Ziehen - Mauszeiger-Verfolgung: Main pollt globalen Cursor (screen.getCursorScreenPoint), Renderer richtet Augen (lookAt-Offset in Kamera-Achsen) + dezent Kopf/Nacken danach aus - Antippen/Streicheln: Klick auf den Avatar -> kurze freudige Reaktion (Laecheln + Kopf-Wackeln) - Ziehen: im Overlay ist der Avatar eine Drag-Region (am Koerper greifen -> Fenster verschieben), Kamera-Drehung (OrbitControls) im Overlay aus; Blasen/Knoepfe bleiben no-drag - neue IPC-Events: lucy:cursor (+ preload onCursor) Co-Authored-By: Claude Opus 4.8 --- client/lucy-desktop/src/main/index.ts | 21 ++++++++- client/lucy-desktop/src/preload/index.ts | 6 +++ client/lucy-desktop/src/renderer/src/App.tsx | 10 ++++- .../src/renderer/src/components/Avatar3D.tsx | 44 ++++++++++++++++--- .../lucy-desktop/src/renderer/src/config.ts | 1 + .../lucy-desktop/src/renderer/src/styles.css | 3 ++ 6 files changed, 75 insertions(+), 10 deletions(-) diff --git a/client/lucy-desktop/src/main/index.ts b/client/lucy-desktop/src/main/index.ts index b982435..a05c1cf 100644 --- a/client/lucy-desktop/src/main/index.ts +++ b/client/lucy-desktop/src/main/index.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell } from "electron" +import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen } from "electron" import { spawn, ChildProcess } from "child_process" import { join } from "path" @@ -20,6 +20,22 @@ let tray: Tray | null = null let ttsProc: ChildProcess | null = null let mode: "full" | "overlay" = "full" let pinned = false +let cursorTimer: ReturnType | null = null + +// MateEngine-Idee: Lucy folgt dem Mauszeiger. Der Zeiger ist auch AUSSERHALB des Fensters (Geistmodus), +// daher global pollen (screen.getCursorScreenPoint) und die Richtung relativ zu Lucys Kopf an den Renderer geben. +function startCursorTracking() { + if (cursorTimer) return + cursorTimer = setInterval(() => { + if (!win || win.isDestroyed() || !win.isVisible()) return + const b = win.getBounds() + const p = screen.getCursorScreenPoint() + const cx = b.x + b.width / 2 + const cy = b.y + b.height * 0.38 // ungefaehre Kopfhoehe im Fenster + win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) }) + }, 40) // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last +} +function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } } async function ensureTtsServer() { try { @@ -150,8 +166,9 @@ app.whenReady().then(() => { createWindow() buildTray() registerHotkey() + startCursorTracking() app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() }) }) app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ }) -app.on("before-quit", () => { stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() }) +app.on("before-quit", () => { stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() }) diff --git a/client/lucy-desktop/src/preload/index.ts b/client/lucy-desktop/src/preload/index.ts index db46276..3a5c51f 100644 --- a/client/lucy-desktop/src/preload/index.ts +++ b/client/lucy-desktop/src/preload/index.ts @@ -25,4 +25,10 @@ contextBridge.exposeInMainWorld("lucy", { ipcRenderer.on("lucy:hotkey", h) return () => ipcRenderer.removeListener("lucy:hotkey", h) }, + // Mauszeiger-Position (relativ zu Lucys Kopf) -> Blickfolgen + onCursor: (cb: (p: { x: number; y: number }) => void) => { + const h = (_e: unknown, p: { x: number; y: number }) => cb(p) + ipcRenderer.on("lucy:cursor", h) + return () => ipcRenderer.removeListener("lucy:cursor", h) + }, }) diff --git a/client/lucy-desktop/src/renderer/src/App.tsx b/client/lucy-desktop/src/renderer/src/App.tsx index e6e46e7..b653f15 100644 --- a/client/lucy-desktop/src/renderer/src/App.tsx +++ b/client/lucy-desktop/src/renderer/src/App.tsx @@ -21,6 +21,8 @@ export default function App() { inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent() const holding = useRef(false) const endRef = useRef(null) + const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen) + const pat = useRef(0) // Zaehler: Antippen/Streicheln des Avatars const [mode, setMode] = useState<"full" | "overlay">("full") const [pinned, setPinned] = useState(false) const [ghost, setGhost] = useState(false) // Geistmodus: Fenster durchklickbar (nur Overlay) @@ -49,6 +51,9 @@ export default function App() { }, [ghost, mode]) useEffect(() => { if (mode !== "overlay") setGhost(false) }, [mode]) // Geistmodus nur im Overlay + // Mauszeiger-Verfolgung (MateEngine-Idee): Position vom Main-Prozess in den Avatar-Blick fuettern + useEffect(() => window.lucy?.onCursor((p) => { cursor.current = p }), []) + // Leertaste = Push-to-talk (halten) useEffect(() => { const isField = (el: EventTarget | null) => @@ -126,10 +131,11 @@ export default function App() {
-
+
{ pat.current += 1 }} + title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
- + {speaking && lastAssistant &&
{lastAssistant}
} {!ready && (
diff --git a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx index d028aee..6284ffe 100644 --- a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx +++ b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx @@ -14,6 +14,8 @@ import type { Emotion } from "../lib/voice/sentiment" type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }> type EmotionRef = MutableRefObject +type CursorRef = MutableRefObject<{ x: number; y: number }> +type NumRef = MutableRefObject // Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md). // Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen, @@ -75,8 +77,9 @@ const EMO_TO_EXPR: Record = { neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed", } -function VrmModel({ url, audioLevel, emotion, status, onError }: { - url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject; onError: (m: string) => void +function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: { + url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject + cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void }) { const [vrm, setVrm] = useState(null) const smooth = useRef>({}) @@ -86,6 +89,10 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: { const gaze = useRef(new Object3D()) // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera) const headPos = useRef(new Vector3()) const camPos = useRef(new Vector3()) + const camRight = useRef(new Vector3()) + const camUp = useRef(new Vector3()) + const patReact = useRef(0) // 0..1 Reaktion aufs Antippen/Streicheln + const lastPat = useRef(0) // Bewegungs-Clips const mixer = useRef(null) const actions = useRef>({}) @@ -182,6 +189,17 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: { } else { applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf } + // MateEngine-Idee: Kopf dreht sich dezent zum Mauszeiger (Augen folgen unten im lookAt-Block) + const cur = cursor?.current + if (cur) { + const hx = Math.max(-1, Math.min(1, cur.x)), hy = Math.max(-1, Math.min(1, cur.y)) + addBone(vrm, "neck", hy * 0.06, hx * 0.10, 0) + addBone(vrm, "head", hy * 0.08, hx * 0.14, 0) + } + // Antippen/Streicheln: kurze freudige Reaktion (Kopf-Wackeln; Laecheln in der Mimik unten) + if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 } + patReact.current *= Math.max(0, 1 - delta * 1.4) + if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06) // Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich". if (vrm.lookAt) { @@ -190,6 +208,15 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: { if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0) const cp = camPos.current.copy(state.camera.position) gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp) + // Augen folgen dem Mauszeiger: Blickpunkt in Kamera-Rechts/Hoch-Richtung verschieben + if (cur) { + const e = state.camera.matrixWorld.elements + camRight.current.set(e[0], e[1], e[2]).normalize() + camUp.current.set(e[4], e[5], e[6]).normalize() + const cx = Math.max(-1.5, Math.min(1.5, cur.x)), cy = Math.max(-1.5, Math.min(1.5, cur.y)) + gaze.current.position.addScaledVector(camRight.current, cx * 1.8) + gaze.current.position.addScaledVector(camUp.current, -cy * 1.8) + } // dezente Blick-Mikrobewegung (Sakkaden) -> das Gesicht wirkt nicht eingefroren const s = sac.current s.t += delta @@ -225,6 +252,8 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: { const expr = brk.current.name === "Relax" ? "happy" : "relaxed" em.setValue(expr, Math.max(smooth.current[expr] ?? 0, brk.current.w * 0.35)) } + // Streichel-Reaktion: Laecheln + if (patReact.current > 0.01) em.setValue("happy", Math.max(smooth.current.happy ?? 0, patReact.current * 0.85)) const b = blink.current b.t += delta if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 } @@ -238,8 +267,9 @@ function VrmModel({ url, audioLevel, emotion, status, onError }: { return vrm ? : null } -export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: { +export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: { url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string + cursor?: CursorRef; pat?: NumRef; controls?: boolean }) { const [err, setErr] = useState(null) const statusRef = useRef(status); statusRef.current = status @@ -249,9 +279,11 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle" }: { - - + + {controls && ( + + )} {err && (
Promise onMode: (cb: (m: "full" | "overlay") => void) => () => void onHotkey: (cb: () => void) => () => void + onCursor: (cb: (p: { x: number; y: number }) => void) => () => void captureScreen: () => Promise listWindows: () => Promise<{ id: string; name: string }[]> captureWindow: (id: string) => Promise diff --git a/client/lucy-desktop/src/renderer/src/styles.css b/client/lucy-desktop/src/renderer/src/styles.css index 6bc079d..7b5fb63 100644 --- a/client/lucy-desktop/src/renderer/src/styles.css +++ b/client/lucy-desktop/src/renderer/src/styles.css @@ -43,6 +43,9 @@ body { } .app.overlay .stage { border: none; background: transparent; } .avatarWrap { flex: 1; min-height: 0; position: relative; overflow: hidden; border-radius: 14px; } +/* Overlay: Lucy am Koerper greifen & ueber den Desktop ziehen (Bedienelemente bleiben no-drag) */ +.app.overlay .avatarWrap { -webkit-app-region: drag; } +.app.overlay .overlayBubbles, .app.overlay .overlayBubbles * { -webkit-app-region: no-drag; } /* --- digitaler Raum (animierter Sci-Fi-Backdrop, Etappe 2B) --- */ .digitalRoom {