diff --git a/client/lucy-desktop/src/main/index.ts b/client/lucy-desktop/src/main/index.ts index a05c1cf..3f73c8b 100644 --- a/client/lucy-desktop/src/main/index.ts +++ b/client/lucy-desktop/src/main/index.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen } from "electron" +import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen, session } from "electron" import { spawn, ChildProcess } from "child_process" import { join } from "path" @@ -33,10 +33,61 @@ function startCursorTracking() { const cx = b.x + b.width / 2 const cy = b.y + b.height * 0.38 // ungefaehre Kopfhoehe im Fenster win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) }) + // Auto-Ducken: im Overlay nach Nicht-Naehe an den Rand; bei Annaeherung an den Rest hervorholen + if (mode === "overlay" && !pinned && !boundsAnim) { + const now = Date.now() + const inWin = p.x >= b.x && p.y >= b.y && p.x <= b.x + b.width && p.y <= b.y + b.height + if (inWin) lastNear = now + if (!tucked && now - lastNear > AUTO_TUCK_MS) tuckToEdge() + else if (tucked) { + const wa = screen.getDisplayNearestPoint({ x: b.x, y: b.y }).workArea + if (p.x >= wa.x + wa.width - SLIVER - 12 && p.y >= b.y - 12 && p.y <= b.y + b.height + 12) { lastNear = now; untuck() } + } + } }, 40) // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last } function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } } +// --- An den Rand ducken (MateEngine-Idee) --- +type Rect = { x: number; y: number; width: number; height: number } +let tucked = false +let prevBounds: Rect | null = null +let lastNear = Date.now() +let boundsAnim: ReturnType | null = null +const SLIVER = 58 // sichtbarer Rest am Rand, wenn geduckt +const AUTO_TUCK_MS = 15000 // nach so langer Nicht-Naehe im Overlay automatisch ducken + +function animateBounds(target: Rect, after?: () => void) { + if (!win) return + if (boundsAnim) { clearInterval(boundsAnim); boundsAnim = null } + const start = win.getBounds(); const t0 = Date.now(); const dur = 260 + boundsAnim = setInterval(() => { + if (!win || win.isDestroyed()) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; return } + const k = Math.min(1, (Date.now() - t0) / dur) + const e = k < 0.5 ? 2 * k * k : 1 - Math.pow(-2 * k + 2, 2) / 2 // easeInOutQuad + win.setBounds({ + x: Math.round(start.x + (target.x - start.x) * e), + y: Math.round(start.y + (target.y - start.y) * e), + width: target.width, height: target.height, + }) + if (k >= 1) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; after?.() } + }, 16) +} +function tuckToEdge() { + if (!win || mode !== "overlay" || tucked) return + prevBounds = win.getBounds() + const wa = screen.getDisplayNearestPoint({ x: prevBounds.x, y: prevBounds.y }).workArea + animateBounds({ x: wa.x + wa.width - SLIVER, y: prevBounds.y, width: prevBounds.width, height: prevBounds.height }) + tucked = true; setTrayMenu() +} +function untuck() { + if (!win || !tucked) return + const t = prevBounds || win.getBounds() + animateBounds({ x: t.x, y: t.y, width: t.width, height: t.height }) + tucked = false; setTrayMenu() +} +function toggleTuck() { if (tucked) untuck(); else tuckToEdge() } + async function ensureTtsServer() { try { const r = await fetch(`${TTS}/health`) @@ -56,6 +107,8 @@ function stopTtsServer() { function applyMode(next: "full" | "overlay") { if (!win) return + // beim Verlassen des Overlays ggf. Duck-Zustand zuruecksetzen (Position wiederherstellen) + if (tucked && next !== "overlay") { tucked = false; if (prevBounds) win.setPosition(prevBounds.x, prevBounds.y) } mode = next const overlay = next === "overlay" const size = overlay ? OVERLAY_SIZE : FULL_SIZE @@ -64,6 +117,7 @@ function applyMode(next: "full" | "overlay") { win.setResizable(!overlay) if (overlay) win.setMinimumSize(280, 360); else win.setMinimumSize(420, 420) win.webContents.send("lucy:mode", next) + lastNear = Date.now(); setTrayMenu() } function createWindow() { @@ -85,6 +139,7 @@ function createWindow() { function showWin() { if (!win) { createWindow(); return } if (win.isMinimized()) win.restore() + if (tucked) untuck() win.show(); win.focus() } @@ -99,17 +154,21 @@ function lucyIcon() { return nativeImage.createFromBitmap(buf, { width: s, height: s }) } +function setTrayMenu() { + if (!tray) return + tray.setContextMenu(Menu.buildFromTemplate([ + { label: "Lucy zeigen", click: showWin }, + { label: "Overlay-Modus", type: "checkbox", checked: mode === "overlay", + click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } }, + { label: tucked ? "Hervorholen" : "An den Rand ducken", enabled: mode === "overlay", click: toggleTuck }, + { type: "separator" }, + { label: "Beenden", click: () => { app.quit() } }, + ])) +} function buildTray() { tray = new Tray(lucyIcon()) tray.setToolTip("Lucy") - const menu = Menu.buildFromTemplate([ - { label: "Lucy zeigen", click: showWin }, - { label: "Overlay-Modus", type: "checkbox", checked: false, - click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } }, - { type: "separator" }, - { label: "Beenden", click: () => { app.quit() } }, - ]) - tray.setContextMenu(menu) + setTrayMenu() tray.on("click", showWin) } @@ -167,8 +226,17 @@ app.whenReady().then(() => { buildTray() registerHotkey() startCursorTracking() + // Tanzen zur Musik: System-Audio (Loopback) fuer getDisplayMedia bereitstellen, ohne System-Picker + session.defaultSession.setDisplayMediaRequestHandler((_req, callback) => { + desktopCapturer.getSources({ types: ["screen"] }) + .then((sources) => callback({ video: sources[0], audio: "loopback" })) + .catch(() => { try { callback({}) } catch { /* */ } }) + }, { useSystemPicker: false }) app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() }) }) app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ }) -app.on("before-quit", () => { stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() }) +app.on("before-quit", () => { + if (boundsAnim) clearInterval(boundsAnim) + stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() +}) diff --git a/client/lucy-desktop/src/renderer/src/App.tsx b/client/lucy-desktop/src/renderer/src/App.tsx index b653f15..4577ace 100644 --- a/client/lucy-desktop/src/renderer/src/App.tsx +++ b/client/lucy-desktop/src/renderer/src/App.tsx @@ -3,6 +3,7 @@ import { Avatar3D } from "./components/Avatar3D" import { AuraGlow } from "./components/AuraGlow" import { ChatMarkdown } from "./components/ChatMarkdown" import { useVoiceAgent } from "./lib/voice/useVoiceAgent" +import { useDanceAudio } from "./lib/voice/useDanceAudio" const FIXED_AVATAR = "/avatar.vrm" @@ -19,6 +20,7 @@ const STATUS_LABEL: Record = { export default function App() { const { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset, inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent() + const { danceLevel, dancing, toggleDance } = useDanceAudio() const holding = useRef(false) const endRef = useRef(null) const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen) @@ -135,7 +137,7 @@ export default function App() { title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
- + {speaking && lastAssistant &&
{lastAssistant}
} {!ready && (
@@ -183,6 +185,10 @@ export default function App() {
)} + {ready && ( + + )} {!overlay && (
{inputMode === "vad" diff --git a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx index 6284ffe..91f5245 100644 --- a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx +++ b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx @@ -77,9 +77,9 @@ const EMO_TO_EXPR: Record = { neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed", } -function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: { +function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onError }: { url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject - cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void + cursor?: CursorRef; pat?: NumRef; dance?: NumRef; onError: (m: string) => void }) { const [vrm, setVrm] = useState(null) const smooth = useRef>({}) @@ -200,6 +200,20 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: { if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 } patReact.current *= Math.max(0, 1 - delta * 1.4) if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06) + // Tanzen zur Musik: rhythmische Ganzkoerper-Bewegung, skaliert mit der Bass-Energie + const dl = dance?.current ?? 0 + if (dl > 0.03) { + const bt = tc * 5.5 + addBone(vrm, "hips", 0, Math.sin(bt * 0.5) * 0.14 * dl, Math.sin(bt) * 0.07 * dl) + addBone(vrm, "spine", 0, Math.sin(bt * 0.5) * 0.06 * dl, Math.sin(bt) * 0.06 * dl) + addBone(vrm, "chest", 0, 0, Math.sin(bt + 0.5) * 0.05 * dl) + addBone(vrm, "head", Math.sin(bt) * 0.05 * dl, Math.sin(bt * 0.5) * 0.05 * dl, Math.sin(bt) * 0.05 * dl) + addBone(vrm, "leftUpperArm", 0, 0, Math.sin(bt) * 0.3 * dl) + addBone(vrm, "rightUpperArm", 0, 0, -Math.sin(bt) * 0.3 * dl) + vrm.scene.position.y = Math.abs(Math.sin(bt)) * 0.05 * dl // Huepfen + } else if (vrm.scene.position.y !== 0) { + vrm.scene.position.y += (0 - vrm.scene.position.y) * Math.min(1, delta * 5) + } // Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich". if (vrm.lookAt) { @@ -267,9 +281,9 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: { return vrm ? : null } -export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: { +export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: { url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string - cursor?: CursorRef; pat?: NumRef; controls?: boolean + cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean }) { const [err, setErr] = useState(null) const statusRef = useRef(status); statusRef.current = status @@ -279,7 +293,7 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pa - + {controls && ( diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/useDanceAudio.ts b/client/lucy-desktop/src/renderer/src/lib/voice/useDanceAudio.ts new file mode 100644 index 0000000..5dde94c --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/lib/voice/useDanceAudio.ts @@ -0,0 +1,48 @@ +import { useCallback, useEffect, useRef, useState } from "react" + +// Tanzen zur Musik (MateEngine-Idee): nimmt den SYSTEM-Ton per Loopback ab (getDisplayMedia, der +// Main-Prozess liefert audio:"loopback") und misst die Bass-Energie. Daraus speist sich danceLevel, +// das der Avatar in eine rhythmische Tanzbewegung umsetzt. Kein Video wird verwendet (Track sofort gestoppt). +export function useDanceAudio() { + const level = useRef(0) + const [dancing, setDancing] = useState(false) + const ctxRef = useRef(null) + const streamRef = useRef(null) + const rafRef = useRef(0) + + const stop = useCallback(() => { + cancelAnimationFrame(rafRef.current) + streamRef.current?.getTracks().forEach((t) => { try { t.stop() } catch { /* */ } }) + streamRef.current = null + ctxRef.current?.close().catch(() => {}); ctxRef.current = null + level.current = 0; setDancing(false) + }, []) + + const start = useCallback(async () => { + try { + // Video wird vom Handler verlangt, brauchen wir aber nicht -> sofort stoppen, nur Audio behalten + const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true }) + stream.getVideoTracks().forEach((t) => t.stop()) + if (stream.getAudioTracks().length === 0) { stream.getTracks().forEach((t) => t.stop()); throw new Error("kein System-Audio") } + const ctx = new (window.AudioContext || (window as any).webkitAudioContext)() + const src = ctx.createMediaStreamSource(stream) + const an = ctx.createAnalyser(); an.fftSize = 256; an.smoothingTimeConstant = 0.7 + src.connect(an) + const freq = new Uint8Array(an.frequencyBinCount) + const tick = () => { + an.getByteFrequencyData(freq) + let s = 0; for (let i = 1; i < 10; i++) s += freq[i] // Bass-/Kick-Bereich + level.current = Math.min(1, (s / 9 / 255) * 1.7) + rafRef.current = requestAnimationFrame(tick) + } + tick() + ctxRef.current = ctx; streamRef.current = stream + setDancing(true) + } catch (e) { console.error("Tanz-Audio:", e); stop() } + }, [stop]) + + const toggleDance = useCallback(() => { dancing ? stop() : void start() }, [dancing, start, stop]) + + useEffect(() => () => stop(), [stop]) + return { danceLevel: level, dancing, toggleDance } +} diff --git a/client/lucy-desktop/src/renderer/src/styles.css b/client/lucy-desktop/src/renderer/src/styles.css index 7b5fb63..90849ff 100644 --- a/client/lucy-desktop/src/renderer/src/styles.css +++ b/client/lucy-desktop/src/renderer/src/styles.css @@ -176,6 +176,15 @@ body { .app.overlay .modeToggle { display: none; } .vadDot { width: 7px; height: 7px; border-radius: 50%; background: #34d399; box-shadow: 0 0 8px #34d399; animation: pulse 1s infinite; } +.danceBtn { + border: 1px solid rgba(167,139,250,0.35); background: rgba(167,139,250,0.12); color: #c4b5fd; + font-size: 11px; padding: 4px 12px; border-radius: 9px; cursor: pointer; -webkit-app-region: no-drag; + transition: background 0.12s, color 0.12s; +} +.danceBtn:hover { background: rgba(167,139,250,0.22); color: #ddd6fe; } +.danceBtn.on { background: rgba(167,139,250,0.3); color: #ede9fe; border-color: rgba(167,139,250,0.6); } +.app.overlay .danceBtn { display: none; } /* im Overlay kein Platz -> nur im Vollfenster */ + .side { width: 320px; flex-shrink: 0; display: flex; flex-direction: column; border: 1px solid rgba(255,255,255,0.07); border-radius: 14px; background: rgba(255,255,255,0.02);