Lucy v2 (MateEngine komplett): an den Rand ducken + Tanzen zur Musik

Rand-ducken: - Overlay-Fenster gleitet nach ~15s Nicht-Naehe an den Bildschirmrand (Sliver sichtbar), kommt zurueck bei Annaeherung; Tray-Menuepunkt "An den Rand ducken/Hervorholen"; weiche Bounds-Animation (easeInOut) - pinned/full unterdruecken Auto-Ducken; showWin/applyMode setzen sauber zurueck Tanzen zur Musik: - Main: setDisplayMediaRequestHandler -> System-Audio per Loopback fuer getDisplayMedia - useDanceAudio: nimmt System-Ton ab, misst Bass-Energie -> danceLevel (Video-Track verworfen) - Avatar: rhythmische Ganzkoerper-Tanzbewegung + Huepfen, skaliert mit der Energie - "Tanzen"-Knopf im Vollfenster Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-30 19:41:43 +02:00
parent c3c7c2ca91
commit 38f8c96b66
5 changed files with 161 additions and 16 deletions
@@ -1,4 +1,4 @@
-import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen } from "electron"
+import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen, session } from "electron"
 import { spawn, ChildProcess } from "child_process"
 import { join } from "path"

@@ -33,10 +33,61 @@ function startCursorTracking() {
    const cx = b.x + b.width / 2
    const cy = b.y + b.height * 0.38   // ungefaehre Kopfhoehe im Fenster
    win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) })
+    // Auto-Ducken: im Overlay nach Nicht-Naehe an den Rand; bei Annaeherung an den Rest hervorholen
+    if (mode === "overlay" && !pinned && !boundsAnim) {
+      const now = Date.now()
+      const inWin = p.x >= b.x && p.y >= b.y && p.x <= b.x + b.width && p.y <= b.y + b.height
+      if (inWin) lastNear = now
+      if (!tucked && now - lastNear > AUTO_TUCK_MS) tuckToEdge()
+      else if (tucked) {
+        const wa = screen.getDisplayNearestPoint({ x: b.x, y: b.y }).workArea
+        if (p.x >= wa.x + wa.width - SLIVER - 12 && p.y >= b.y - 12 && p.y <= b.y + b.height + 12) { lastNear = now; untuck() }
+      }
+    }
  }, 40)   // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last
 }
 function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } }

+// --- An den Rand ducken (MateEngine-Idee) ---
+type Rect = { x: number; y: number; width: number; height: number }
+let tucked = false
+let prevBounds: Rect | null = null
+let lastNear = Date.now()
+let boundsAnim: ReturnType<typeof setInterval> | null = null
+const SLIVER = 58              // sichtbarer Rest am Rand, wenn geduckt
+const AUTO_TUCK_MS = 15000     // nach so langer Nicht-Naehe im Overlay automatisch ducken
+
+function animateBounds(target: Rect, after?: () => void) {
+  if (!win) return
+  if (boundsAnim) { clearInterval(boundsAnim); boundsAnim = null }
+  const start = win.getBounds(); const t0 = Date.now(); const dur = 260
+  boundsAnim = setInterval(() => {
+    if (!win || win.isDestroyed()) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; return }
+    const k = Math.min(1, (Date.now() - t0) / dur)
+    const e = k < 0.5 ? 2 * k * k : 1 - Math.pow(-2 * k + 2, 2) / 2   // easeInOutQuad
+    win.setBounds({
+      x: Math.round(start.x + (target.x - start.x) * e),
+      y: Math.round(start.y + (target.y - start.y) * e),
+      width: target.width, height: target.height,
+    })
+    if (k >= 1) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; after?.() }
+  }, 16)
+}
+function tuckToEdge() {
+  if (!win || mode !== "overlay" || tucked) return
+  prevBounds = win.getBounds()
+  const wa = screen.getDisplayNearestPoint({ x: prevBounds.x, y: prevBounds.y }).workArea
+  animateBounds({ x: wa.x + wa.width - SLIVER, y: prevBounds.y, width: prevBounds.width, height: prevBounds.height })
+  tucked = true; setTrayMenu()
+}
+function untuck() {
+  if (!win || !tucked) return
+  const t = prevBounds || win.getBounds()
+  animateBounds({ x: t.x, y: t.y, width: t.width, height: t.height })
+  tucked = false; setTrayMenu()
+}
+function toggleTuck() { if (tucked) untuck(); else tuckToEdge() }
+
 async function ensureTtsServer() {
  try {
    const r = await fetch(`${TTS}/health`)
@@ -56,6 +107,8 @@ function stopTtsServer() {

 function applyMode(next: "full" | "overlay") {
  if (!win) return
+  // beim Verlassen des Overlays ggf. Duck-Zustand zuruecksetzen (Position wiederherstellen)
+  if (tucked && next !== "overlay") { tucked = false; if (prevBounds) win.setPosition(prevBounds.x, prevBounds.y) }
  mode = next
  const overlay = next === "overlay"
  const size = overlay ? OVERLAY_SIZE : FULL_SIZE
@@ -64,6 +117,7 @@ function applyMode(next: "full" | "overlay") {
  win.setResizable(!overlay)
  if (overlay) win.setMinimumSize(280, 360); else win.setMinimumSize(420, 420)
  win.webContents.send("lucy:mode", next)
+  lastNear = Date.now(); setTrayMenu()
 }

 function createWindow() {
@@ -85,6 +139,7 @@ function createWindow() {
 function showWin() {
  if (!win) { createWindow(); return }
  if (win.isMinimized()) win.restore()
+  if (tucked) untuck()
  win.show(); win.focus()
 }

@@ -99,17 +154,21 @@ function lucyIcon() {
  return nativeImage.createFromBitmap(buf, { width: s, height: s })
 }

+function setTrayMenu() {
+  if (!tray) return
+  tray.setContextMenu(Menu.buildFromTemplate([
+    { label: "Lucy zeigen", click: showWin },
+    { label: "Overlay-Modus", type: "checkbox", checked: mode === "overlay",
+      click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } },
+    { label: tucked ? "Hervorholen" : "An den Rand ducken", enabled: mode === "overlay", click: toggleTuck },
+    { type: "separator" },
+    { label: "Beenden", click: () => { app.quit() } },
+  ]))
+}
 function buildTray() {
  tray = new Tray(lucyIcon())
  tray.setToolTip("Lucy")
-  const menu = Menu.buildFromTemplate([
-    { label: "Lucy zeigen", click: showWin },
-    { label: "Overlay-Modus", type: "checkbox", checked: false,
-      click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } },
-    { type: "separator" },
-    { label: "Beenden", click: () => { app.quit() } },
-  ])
-  tray.setContextMenu(menu)
+  setTrayMenu()
  tray.on("click", showWin)
 }

@@ -167,8 +226,17 @@ app.whenReady().then(() => {
  buildTray()
  registerHotkey()
  startCursorTracking()
+  // Tanzen zur Musik: System-Audio (Loopback) fuer getDisplayMedia bereitstellen, ohne System-Picker
+  session.defaultSession.setDisplayMediaRequestHandler((_req, callback) => {
+    desktopCapturer.getSources({ types: ["screen"] })
+      .then((sources) => callback({ video: sources[0], audio: "loopback" }))
+      .catch(() => { try { callback({}) } catch { /* */ } })
+  }, { useSystemPicker: false })
  app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() })
 })

 app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ })
-app.on("before-quit", () => { stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy() })
+app.on("before-quit", () => {
+  if (boundsAnim) clearInterval(boundsAnim)
+  stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy()
+})
@@ -3,6 +3,7 @@ import { Avatar3D } from "./components/Avatar3D"
 import { AuraGlow } from "./components/AuraGlow"
 import { ChatMarkdown } from "./components/ChatMarkdown"
 import { useVoiceAgent } from "./lib/voice/useVoiceAgent"
+import { useDanceAudio } from "./lib/voice/useDanceAudio"

 const FIXED_AVATAR = "/avatar.vrm"

@@ -19,6 +20,7 @@ const STATUS_LABEL: Record<string, string> = {
 export default function App() {
  const { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset,
          inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
+  const { danceLevel, dancing, toggleDance } = useDanceAudio()
  const holding = useRef(false)
  const endRef = useRef<HTMLDivElement>(null)
  const cursor = useRef({ x: 0, y: 0 })   // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
@@ -135,7 +137,7 @@ export default function App() {
               title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
            <div className="digitalRoom"><div className="dust" /></div>
            <AuraGlow status={status} audioLevel={audioLevel} />
-            <Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} controls={!overlay} />
+            <Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} dance={danceLevel} controls={!overlay} />
            {speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
            {!ready && (
              <div className="warmOverlay">
@@ -183,6 +185,10 @@ export default function App() {
                <button className={inputMode === "vad" ? "on" : ""} onClick={() => setInputMode("vad")}>Freisprechen</button>
              </div>
            )}
+            {ready && (
+              <button className={"danceBtn" + (dancing ? " on" : "")} onClick={toggleDance}
+                title="Zur Musik tanzen (nimmt den System-Ton ab)">{dancing ? "⏹ Tanz aus" : "💃 Tanzen"}</button>
+            )}
            {!overlay && (
              <div className="hint">
                {inputMode === "vad"
@@ -77,9 +77,9 @@ const EMO_TO_EXPR: Record<Emotion, string | null> = {
  neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
 }

-function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
+function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onError }: {
  url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
-  cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void
+  cursor?: CursorRef; pat?: NumRef; dance?: NumRef; onError: (m: string) => void
 }) {
  const [vrm, setVrm] = useState<VRM | null>(null)
  const smooth = useRef<Record<string, number>>({})
@@ -200,6 +200,20 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
    if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
    patReact.current *= Math.max(0, 1 - delta * 1.4)
    if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
+    // Tanzen zur Musik: rhythmische Ganzkoerper-Bewegung, skaliert mit der Bass-Energie
+    const dl = dance?.current ?? 0
+    if (dl > 0.03) {
+      const bt = tc * 5.5
+      addBone(vrm, "hips", 0, Math.sin(bt * 0.5) * 0.14 * dl, Math.sin(bt) * 0.07 * dl)
+      addBone(vrm, "spine", 0, Math.sin(bt * 0.5) * 0.06 * dl, Math.sin(bt) * 0.06 * dl)
+      addBone(vrm, "chest", 0, 0, Math.sin(bt + 0.5) * 0.05 * dl)
+      addBone(vrm, "head", Math.sin(bt) * 0.05 * dl, Math.sin(bt * 0.5) * 0.05 * dl, Math.sin(bt) * 0.05 * dl)
+      addBone(vrm, "leftUpperArm", 0, 0, Math.sin(bt) * 0.3 * dl)
+      addBone(vrm, "rightUpperArm", 0, 0, -Math.sin(bt) * 0.3 * dl)
+      vrm.scene.position.y = Math.abs(Math.sin(bt)) * 0.05 * dl   // Huepfen
+    } else if (vrm.scene.position.y !== 0) {
+      vrm.scene.position.y += (0 - vrm.scene.position.y) * Math.min(1, delta * 5)
+    }

    // Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
    if (vrm.lookAt) {
@@ -267,9 +281,9 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
  return vrm ? <primitive object={vrm.scene} /> : null
 }

-export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: {
+export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: {
  url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
-  cursor?: CursorRef; pat?: NumRef; controls?: boolean
+  cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean
 }) {
  const [err, setErr] = useState<string | null>(null)
  const statusRef = useRef(status); statusRef.current = status
@@ -279,7 +293,7 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pa
        <ambientLight intensity={0.85} />
        <directionalLight position={[1, 2, 2]} intensity={1.1} />
        <directionalLight position={[-1, 1, -1]} intensity={0.4} />
-        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} onError={setErr} />
+        <VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} dance={dance} onError={setErr} />
        {controls && (
          <OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
            minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
@@ -0,0 +1,48 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+
+// Tanzen zur Musik (MateEngine-Idee): nimmt den SYSTEM-Ton per Loopback ab (getDisplayMedia, der
+// Main-Prozess liefert audio:"loopback") und misst die Bass-Energie. Daraus speist sich danceLevel,
+// das der Avatar in eine rhythmische Tanzbewegung umsetzt. Kein Video wird verwendet (Track sofort gestoppt).
+export function useDanceAudio() {
+  const level = useRef(0)
+  const [dancing, setDancing] = useState(false)
+  const ctxRef = useRef<AudioContext | null>(null)
+  const streamRef = useRef<MediaStream | null>(null)
+  const rafRef = useRef(0)
+
+  const stop = useCallback(() => {
+    cancelAnimationFrame(rafRef.current)
+    streamRef.current?.getTracks().forEach((t) => { try { t.stop() } catch { /* */ } })
+    streamRef.current = null
+    ctxRef.current?.close().catch(() => {}); ctxRef.current = null
+    level.current = 0; setDancing(false)
+  }, [])
+
+  const start = useCallback(async () => {
+    try {
+      // Video wird vom Handler verlangt, brauchen wir aber nicht -> sofort stoppen, nur Audio behalten
+      const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true })
+      stream.getVideoTracks().forEach((t) => t.stop())
+      if (stream.getAudioTracks().length === 0) { stream.getTracks().forEach((t) => t.stop()); throw new Error("kein System-Audio") }
+      const ctx = new (window.AudioContext || (window as any).webkitAudioContext)()
+      const src = ctx.createMediaStreamSource(stream)
+      const an = ctx.createAnalyser(); an.fftSize = 256; an.smoothingTimeConstant = 0.7
+      src.connect(an)
+      const freq = new Uint8Array(an.frequencyBinCount)
+      const tick = () => {
+        an.getByteFrequencyData(freq)
+        let s = 0; for (let i = 1; i < 10; i++) s += freq[i]   // Bass-/Kick-Bereich
+        level.current = Math.min(1, (s / 9 / 255) * 1.7)
+        rafRef.current = requestAnimationFrame(tick)
+      }
+      tick()
+      ctxRef.current = ctx; streamRef.current = stream
+      setDancing(true)
+    } catch (e) { console.error("Tanz-Audio:", e); stop() }
+  }, [stop])
+
+  const toggleDance = useCallback(() => { dancing ? stop() : void start() }, [dancing, start, stop])
+
+  useEffect(() => () => stop(), [stop])
+  return { danceLevel: level, dancing, toggleDance }
+}
@@ -176,6 +176,15 @@ body {
 .app.overlay .modeToggle { display: none; }
 .vadDot { width: 7px; height: 7px; border-radius: 50%; background: #34d399; box-shadow: 0 0 8px #34d399; animation: pulse 1s infinite; }

+.danceBtn {
+  border: 1px solid rgba(167,139,250,0.35); background: rgba(167,139,250,0.12); color: #c4b5fd;
+  font-size: 11px; padding: 4px 12px; border-radius: 9px; cursor: pointer; -webkit-app-region: no-drag;
+  transition: background 0.12s, color 0.12s;
+}
+.danceBtn:hover { background: rgba(167,139,250,0.22); color: #ddd6fe; }
+.danceBtn.on { background: rgba(167,139,250,0.3); color: #ede9fe; border-color: rgba(167,139,250,0.6); }
+.app.overlay .danceBtn { display: none; }   /* im Overlay kein Platz -> nur im Vollfenster */
+
 .side {
  width: 320px; flex-shrink: 0; display: flex; flex-direction: column;
  border: 1px solid rgba(255,255,255,0.07); border-radius: 14px; background: rgba(255,255,255,0.02);