Lucy v2 (MateEngine komplett): an den Rand ducken + Tanzen zur Musik

Rand-ducken:
- Overlay-Fenster gleitet nach ~15s Nicht-Naehe an den Bildschirmrand (Sliver sichtbar), kommt zurueck
  bei Annaeherung; Tray-Menuepunkt "An den Rand ducken/Hervorholen"; weiche Bounds-Animation (easeInOut)
- pinned/full unterdruecken Auto-Ducken; showWin/applyMode setzen sauber zurueck

Tanzen zur Musik:
- Main: setDisplayMediaRequestHandler -> System-Audio per Loopback fuer getDisplayMedia
- useDanceAudio: nimmt System-Ton ab, misst Bass-Energie -> danceLevel (Video-Track verworfen)
- Avatar: rhythmische Ganzkoerper-Tanzbewegung + Huepfen, skaliert mit der Energie
- "Tanzen"-Knopf im Vollfenster

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

client/lucy-desktop/src/main/index.ts

@@ -1,4 +1,4 @@
-import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen } from "electron"
+import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen, session } from "electron"
 import { spawn, ChildProcess } from "child_process"
 import { join } from "path"
 
@@ -33,10 +33,61 @@ function startCursorTracking() {
     const cx = b.x + b.width / 2
     const cy = b.y + b.height * 0.38   // ungefaehre Kopfhoehe im Fenster
     win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) })
+    // Auto-Ducken: im Overlay nach Nicht-Naehe an den Rand; bei Annaeherung an den Rest hervorholen
+    if (mode === "overlay" && !pinned && !boundsAnim) {
+      const now = Date.now()
+      const inWin = p.x >= b.x && p.y >= b.y && p.x <= b.x + b.width && p.y <= b.y + b.height
+      if (inWin) lastNear = now
+      if (!tucked && now - lastNear > AUTO_TUCK_MS) tuckToEdge()
+      else if (tucked) {
+        const wa = screen.getDisplayNearestPoint({ x: b.x, y: b.y }).workArea
+        if (p.x >= wa.x + wa.width - SLIVER - 12 && p.y >= b.y - 12 && p.y <= b.y + b.height + 12) { lastNear = now; untuck() }
+      }
+    }
   }, 40)   // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last
 }
 function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } }
 
+// --- An den Rand ducken (MateEngine-Idee) ---
+type Rect = { x: number; y: number; width: number; height: number }
+let tucked = false
+let prevBounds: Rect | null = null
+let lastNear = Date.now()
+let boundsAnim: ReturnType<typeof setInterval> | null = null
+const SLIVER = 58              // sichtbarer Rest am Rand, wenn geduckt
+const AUTO_TUCK_MS = 15000     // nach so langer Nicht-Naehe im Overlay automatisch ducken
+
+function animateBounds(target: Rect, after?: () => void) {
+  if (!win) return
+  if (boundsAnim) { clearInterval(boundsAnim); boundsAnim = null }
+  const start = win.getBounds(); const t0 = Date.now(); const dur = 260
+  boundsAnim = setInterval(() => {
+    if (!win || win.isDestroyed()) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; return }
+    const k = Math.min(1, (Date.now() - t0) / dur)
+    const e = k < 0.5 ? 2 * k * k : 1 - Math.pow(-2 * k + 2, 2) / 2   // easeInOutQuad
+    win.setBounds({
+      x: Math.round(start.x + (target.x - start.x) * e),
+      y: Math.round(start.y + (target.y - start.y) * e),
+      width: target.width, height: target.height,
+    })
+    if (k >= 1) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; after?.() }
+  }, 16)
+}
+function tuckToEdge() {
+  if (!win || mode !== "overlay" || tucked) return
+  prevBounds = win.getBounds()
+  const wa = screen.getDisplayNearestPoint({ x: prevBounds.x, y: prevBounds.y }).workArea
+  animateBounds({ x: wa.x + wa.width - SLIVER, y: prevBounds.y, width: prevBounds.width, height: prevBounds.height })
+  tucked = true; setTrayMenu()
+}
+function untuck() {
+  if (!win || !tucked) return
+  const t = prevBounds || win.getBounds()
+  animateBounds({ x: t.x, y: t.y, width: t.width, height: t.height })
+  tucked = false; setTrayMenu()
+}
+function toggleTuck() { if (tucked) untuck(); else tuckToEdge() }
+
 async function ensureTtsServer() {
   try {
     const r = await fetch(`${TTS}/health`)
@@ -56,6 +107,8 @@ function stopTtsServer() {
 
 function applyMode(next: "full" | "overlay") {
   if (!win) return
+  // beim Verlassen des Overlays ggf. Duck-Zustand zuruecksetzen (Position wiederherstellen)
+  if (tucked && next !== "overlay") { tucked = false; if (prevBounds) win.setPosition(prevBounds.x, prevBounds.y) }
   mode = next
   const overlay = next === "overlay"
   const size = overlay ? OVERLAY_SIZE : FULL_SIZE
@@ -64,6 +117,7 @@ function applyMode(next: "full" | "overlay") {
   win.setResizable(!overlay)
   if (overlay) win.setMinimumSize(280, 360); else win.setMinimumSize(420, 420)
   win.webContents.send("lucy:mode", next)
+  lastNear = Date.now(); setTrayMenu()
 }
 
 function createWindow() {
@@ -85,6 +139,7 @@ function createWindow() {
 function showWin() {
   if (!win) { createWindow(); return }
   if (win.isMinimized()) win.restore()
+  if (tucked) untuck()
   win.show(); win.focus()
 }
 
@@ -99,17 +154,21 @@ function lucyIcon() {
   return nativeImage.createFromBitmap(buf, { width: s, height: s })
 }
 
+function setTrayMenu() {
+  if (!tray) return
+  tray.setContextMenu(Menu.buildFromTemplate([
+    { label: "Lucy zeigen", click: showWin },
+    { label: "Overlay-Modus", type: "checkbox", checked: mode === "overlay",
+      click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } },
+    { label: tucked ? "Hervorholen" : "An den Rand ducken", enabled: mode === "overlay", click: toggleTuck },
+    { type: "separator" },
+    { label: "Beenden", click: () => { app.quit() } },
+  ]))
+}
 function buildTray() {
   tray = new Tray(lucyIcon())
   tray.setToolTip("Lucy")
-  const menu = Menu.buildFromTemplate([
+  setTrayMenu()
-    { label: "Lucy zeigen", click: showWin },
-    { label: "Overlay-Modus", type: "checkbox", checked: false,
-      click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } },
-    { type: "separator" },
-    { label: "Beenden", click: () => { app.quit() } },
-  ])
-  tray.setContextMenu(menu)
   tray.on("click", showWin)
 }
 
@@ -167,8 +226,17 @@ app.whenReady().then(() => {
   buildTray()
   registerHotkey()
   startCursorTracking()
+  // Tanzen zur Musik: System-Audio (Loopback) fuer getDisplayMedia bereitstellen, ohne System-Picker
+  session.defaultSession.setDisplayMediaRequestHandler((_req, callback) => {
+    desktopCapturer.getSources({ types: ["screen"] })
+      .then((sources) => callback({ video: sources[0], audio: "loopback" }))
+      .catch(() => { try { callback({}) } catch { /* */ } })
+  }, { useSystemPicker: false })
   app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() })
 })
 
 app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ })
-app.on("before-quit", () => { stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() })
+app.on("before-quit", () => {
+  if (boundsAnim) clearInterval(boundsAnim)
+  stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy()
+})

client/lucy-desktop/src/renderer/src/App.tsx

@@ -3,6 +3,7 @@ import { Avatar3D } from "./components/Avatar3D"
 import { AuraGlow } from "./components/AuraGlow"
 import { ChatMarkdown } from "./components/ChatMarkdown"
 import { useVoiceAgent } from "./lib/voice/useVoiceAgent"
+import { useDanceAudio } from "./lib/voice/useDanceAudio"
 
 const FIXED_AVATAR = "/avatar.vrm"
 
@@ -19,6 +20,7 @@ const STATUS_LABEL: Record<string, string> = {
 export default function App() {
   const { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset,
           inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
+  const { danceLevel, dancing, toggleDance } = useDanceAudio()
   const holding = useRef(false)
   const endRef = useRef<HTMLDivElement>(null)
   const cursor = useRef({ x: 0, y: 0 })   // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
@@ -135,7 +137,7 @@ export default function App() {
                title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
             <div className="digitalRoom"><div className="dust" /></div>
             <AuraGlow status={status} audioLevel={audioLevel} />
-            <Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} controls={!overlay} />
+            <Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} dance={danceLevel} controls={!overlay} />
             {speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
             {!ready && (
               <div className="warmOverlay">
@@ -183,6 +185,10 @@ export default function App() {
                 <button className={inputMode === "vad" ? "on" : ""} onClick={() => setInputMode("vad")}>Freisprechen</button>
               </div>
             )}
+            {ready && (
+              <button className={"danceBtn" + (dancing ? " on" : "")} onClick={toggleDance}
+                title="Zur Musik tanzen (nimmt den System-Ton ab)">{dancing ? "⏹ Tanz aus" : "💃 Tanzen"}</button>
+            )}
             {!overlay && (
               <div className="hint">
                 {inputMode === "vad"

client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx

@@ -77,9 +77,9 @@ const EMO_TO_EXPR: Record<Emotion, string | null> = {
   neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
 }
 
-function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
+function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onError }: {
   url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
-  cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void
+  cursor?: CursorRef; pat?: NumRef; dance?: NumRef; onError: (m: string) => void
 }) {
   const [vrm, setVrm] = useState<VRM | null>(null)
   const smooth = useRef<Record<string, number>>({})
@@ -200,6 +200,20 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
     if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
     patReact.current *= Math.max(0, 1 - delta * 1.4)
     if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
+    // Tanzen zur Musik: rhythmische Ganzkoerper-Bewegung, skaliert mit der Bass-Energie
+    const dl = dance?.current ?? 0
+    if (dl > 0.03) {
+      const bt = tc * 5.5
+      addBone(vrm, "hips", 0, Math.sin(bt * 0.5) * 0.14 * dl, Math.sin(bt) * 0.07 * dl)
+      addBone(vrm, "spine", 0, Math.sin(bt * 0.5) * 0.06 * dl, Math.sin(bt) * 0.06 * dl)
+      addBone(vrm, "chest", 0, 0, Math.sin(bt + 0.5) * 0.05 * dl)
+      addBone(vrm, "head", Math.sin(bt) * 0.05 * dl, Math.sin(bt * 0.5) * 0.05 * dl, Math.sin(bt) * 0.05 * dl)
+      addBone(vrm, "leftUpperArm", 0, 0, Math.sin(bt) * 0.3 * dl)
+      addBone(vrm, "rightUpperArm", 0, 0, -Math.sin(bt) * 0.3 * dl)
+      vrm.scene.position.y = Math.abs(Math.sin(bt)) * 0.05 * dl   // Huepfen
+    } else if (vrm.scene.position.y !== 0) {
+      vrm.scene.position.y += (0 - vrm.scene.position.y) * Math.min(1, delta * 5)
+    }
 
     // Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
     if (vrm.lookAt) {
@@ -267,9 +281,9 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
   return vrm ? <primitive object={vrm.scene} /> : null
 }
 
-export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: {
+export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: {
   url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
-  cursor?: CursorRef; pat?: NumRef; controls?: boolean
+  cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean
 }) {
   const [err, setErr] = useState<string | null>(null)
   const statusRef = useRef(status); statusRef.current = status
@@ -279,7 +293,7 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pa
         <ambientLight intensity={0.85} />
         <directionalLight position={[1, 2, 2]} intensity={1.1} />
         <directionalLight position={[-1, 1, -1]} intensity={0.4} />
-        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} onError={setErr} />
+        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} dance={dance} onError={setErr} />
         {controls && (
           <OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
             minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />

client/lucy-desktop/src/renderer/src/lib/voice/useDanceAudio.ts

@@ -0,0 +1,48 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+
+// Tanzen zur Musik (MateEngine-Idee): nimmt den SYSTEM-Ton per Loopback ab (getDisplayMedia, der
+// Main-Prozess liefert audio:"loopback") und misst die Bass-Energie. Daraus speist sich danceLevel,
+// das der Avatar in eine rhythmische Tanzbewegung umsetzt. Kein Video wird verwendet (Track sofort gestoppt).
+export function useDanceAudio() {
+  const level = useRef(0)
+  const [dancing, setDancing] = useState(false)
+  const ctxRef = useRef<AudioContext | null>(null)
+  const streamRef = useRef<MediaStream | null>(null)
+  const rafRef = useRef(0)
+
+  const stop = useCallback(() => {
+    cancelAnimationFrame(rafRef.current)
+    streamRef.current?.getTracks().forEach((t) => { try { t.stop() } catch { /* */ } })
+    streamRef.current = null
+    ctxRef.current?.close().catch(() => {}); ctxRef.current = null
+    level.current = 0; setDancing(false)
+  }, [])
+
+  const start = useCallback(async () => {
+    try {
+      // Video wird vom Handler verlangt, brauchen wir aber nicht -> sofort stoppen, nur Audio behalten
+      const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true })
+      stream.getVideoTracks().forEach((t) => t.stop())
+      if (stream.getAudioTracks().length === 0) { stream.getTracks().forEach((t) => t.stop()); throw new Error("kein System-Audio") }
+      const ctx = new (window.AudioContext || (window as any).webkitAudioContext)()
+      const src = ctx.createMediaStreamSource(stream)
+      const an = ctx.createAnalyser(); an.fftSize = 256; an.smoothingTimeConstant = 0.7
+      src.connect(an)
+      const freq = new Uint8Array(an.frequencyBinCount)
+      const tick = () => {
+        an.getByteFrequencyData(freq)
+        let s = 0; for (let i = 1; i < 10; i++) s += freq[i]   // Bass-/Kick-Bereich
+        level.current = Math.min(1, (s / 9 / 255) * 1.7)
+        rafRef.current = requestAnimationFrame(tick)
+      }
+      tick()
+      ctxRef.current = ctx; streamRef.current = stream
+      setDancing(true)
+    } catch (e) { console.error("Tanz-Audio:", e); stop() }
+  }, [stop])
+
+  const toggleDance = useCallback(() => { dancing ? stop() : void start() }, [dancing, start, stop])
+
+  useEffect(() => () => stop(), [stop])
+  return { danceLevel: level, dancing, toggleDance }
+}

client/lucy-desktop/src/renderer/src/styles.css

@@ -176,6 +176,15 @@ body {
 .app.overlay .modeToggle { display: none; }
 .vadDot { width: 7px; height: 7px; border-radius: 50%; background: #34d399; box-shadow: 0 0 8px #34d399; animation: pulse 1s infinite; }
 
+.danceBtn {
+  border: 1px solid rgba(167,139,250,0.35); background: rgba(167,139,250,0.12); color: #c4b5fd;
+  font-size: 11px; padding: 4px 12px; border-radius: 9px; cursor: pointer; -webkit-app-region: no-drag;
+  transition: background 0.12s, color 0.12s;
+}
+.danceBtn:hover { background: rgba(167,139,250,0.22); color: #ddd6fe; }
+.danceBtn.on { background: rgba(167,139,250,0.3); color: #ede9fe; border-color: rgba(167,139,250,0.6); }
+.app.overlay .danceBtn { display: none; }   /* im Overlay kein Platz -> nur im Vollfenster */
+
 .side {
   width: 320px; flex-shrink: 0; display: flex; flex-direction: column;
   border: 1px solid rgba(255,255,255,0.07); border-radius: 14px; background: rgba(255,255,255,0.02);