Hitonabi/mission-control-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lucy v2 (MateEngine komplett): an den Rand ducken + Tanzen zur Musik

Browse Source 
Rand-ducken:
- Overlay-Fenster gleitet nach ~15s Nicht-Naehe an den Bildschirmrand (Sliver sichtbar), kommt zurueck
  bei Annaeherung; Tray-Menuepunkt "An den Rand ducken/Hervorholen"; weiche Bounds-Animation (easeInOut)
- pinned/full unterdruecken Auto-Ducken; showWin/applyMode setzen sauber zurueck

Tanzen zur Musik:
- Main: setDisplayMediaRequestHandler -> System-Audio per Loopback fuer getDisplayMedia
- useDanceAudio: nimmt System-Ton ab, misst Bass-Energie -> danceLevel (Video-Track verworfen)
- Avatar: rhythmische Ganzkoerper-Tanzbewegung + Huepfen, skaliert mit der Energie
- "Tanzen"-Knopf im Vollfenster

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
2026-06-30 19:41:43 +02:00
parent c3c7c2ca91
commit 38f8c96b66
5 changed files with 161 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/lucy-desktop/src/renderer/src/App.tsx
+7 -1
View File
@@ -3,6 +3,7 @@ import { Avatar3D } from "./components/Avatar3D"
import { AuraGlow } from "./components/AuraGlow"
import { ChatMarkdown } from "./components/ChatMarkdown"
import { useVoiceAgent } from "./lib/voice/useVoiceAgent"
import { useDanceAudio } from "./lib/voice/useDanceAudio"
const FIXED_AVATAR = "/avatar.vrm"
@@ -19,6 +20,7 @@ const STATUS_LABEL: Record<string, string> = {
export default function App() {
const { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset,
inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
const { danceLevel, dancing, toggleDance } = useDanceAudio()
const holding = useRef(false)
const endRef = useRef<HTMLDivElement>(null)
const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
@@ -135,7 +137,7 @@ export default function App() {
title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
<div className="digitalRoom"><div className="dust" /></div>
<AuraGlow status={status} audioLevel={audioLevel} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} controls={!overlay} />
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} dance={danceLevel} controls={!overlay} />
{speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
{!ready && (
<div className="warmOverlay">
@@ -183,6 +185,10 @@ export default function App() {
<button className={inputMode === "vad" ? "on" : ""} onClick={() => setInputMode("vad")}>Freisprechen</button>
</div>
)}
{ready && (
<button className={"danceBtn" + (dancing ? " on" : "")} onClick={toggleDance}
title="Zur Musik tanzen (nimmt den System-Ton ab)">{dancing ? "⏹ Tanz aus" : "💃 Tanzen"}</button>
)}
{!overlay && (
<div className="hint">
{inputMode === "vad"
client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx
+19 -5
View File
@@ -77,9 +77,9 @@ const EMO_TO_EXPR: Record<Emotion, string | null> = {
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
}
function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onError }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
cursor?: CursorRef; pat?: NumRef; onError: (m: string) => void
cursor?: CursorRef; pat?: NumRef; dance?: NumRef; onError: (m: string) => void
}) {
const [vrm, setVrm] = useState<VRM | null>(null)
const smooth = useRef<Record<string, number>>({})
@@ -200,6 +200,20 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
patReact.current *= Math.max(0, 1 - delta * 1.4)
if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
// Tanzen zur Musik: rhythmische Ganzkoerper-Bewegung, skaliert mit der Bass-Energie
const dl = dance?.current ?? 0
if (dl > 0.03) {
const bt = tc * 5.5
addBone(vrm, "hips", 0, Math.sin(bt * 0.5) * 0.14 * dl, Math.sin(bt) * 0.07 * dl)
addBone(vrm, "spine", 0, Math.sin(bt * 0.5) * 0.06 * dl, Math.sin(bt) * 0.06 * dl)
addBone(vrm, "chest", 0, 0, Math.sin(bt + 0.5) * 0.05 * dl)
addBone(vrm, "head", Math.sin(bt) * 0.05 * dl, Math.sin(bt * 0.5) * 0.05 * dl, Math.sin(bt) * 0.05 * dl)
addBone(vrm, "leftUpperArm", 0, 0, Math.sin(bt) * 0.3 * dl)
addBone(vrm, "rightUpperArm", 0, 0, -Math.sin(bt) * 0.3 * dl)
vrm.scene.position.y = Math.abs(Math.sin(bt)) * 0.05 * dl // Huepfen
} else if (vrm.scene.position.y !== 0) {
vrm.scene.position.y += (0 - vrm.scene.position.y) * Math.min(1, delta * 5)
}
// Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
if (vrm.lookAt) {
@@ -267,9 +281,9 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, onError }: {
return vrm ? <primitive object={vrm.scene} /> : null
}
export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, controls = true }: {
export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: {
url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
cursor?: CursorRef; pat?: NumRef; controls?: boolean
cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean
}) {
const [err, setErr] = useState<string | null>(null)
const statusRef = useRef(status); statusRef.current = status
@@ -279,7 +293,7 @@ export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pa
<ambientLight intensity={0.85} />
<directionalLight position={[1, 2, 2]} intensity={1.1} />
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} onError={setErr} />
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} dance={dance} onError={setErr} />
{controls && (
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
client/lucy-desktop/src/renderer/src/lib/voice/useDanceAudio.ts
+48
View File
@@ -0,0 +1,48 @@
import { useCallback, useEffect, useRef, useState } from "react"
// Tanzen zur Musik (MateEngine-Idee): nimmt den SYSTEM-Ton per Loopback ab (getDisplayMedia, der
// Main-Prozess liefert audio:"loopback") und misst die Bass-Energie. Daraus speist sich danceLevel,
// das der Avatar in eine rhythmische Tanzbewegung umsetzt. Kein Video wird verwendet (Track sofort gestoppt).
export function useDanceAudio() {
const level = useRef(0)
const [dancing, setDancing] = useState(false)
const ctxRef = useRef<AudioContext | null>(null)
const streamRef = useRef<MediaStream | null>(null)
const rafRef = useRef(0)
const stop = useCallback(() => {
cancelAnimationFrame(rafRef.current)
streamRef.current?.getTracks().forEach((t) => { try { t.stop() } catch { /* */ } })
streamRef.current = null
ctxRef.current?.close().catch(() => {}); ctxRef.current = null
level.current = 0; setDancing(false)
}, [])
const start = useCallback(async () => {
try {
// Video wird vom Handler verlangt, brauchen wir aber nicht -> sofort stoppen, nur Audio behalten
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true })
stream.getVideoTracks().forEach((t) => t.stop())
if (stream.getAudioTracks().length === 0) { stream.getTracks().forEach((t) => t.stop()); throw new Error("kein System-Audio") }
const ctx = new (window.AudioContext || (window as any).webkitAudioContext)()
const src = ctx.createMediaStreamSource(stream)
const an = ctx.createAnalyser(); an.fftSize = 256; an.smoothingTimeConstant = 0.7
src.connect(an)
const freq = new Uint8Array(an.frequencyBinCount)
const tick = () => {
an.getByteFrequencyData(freq)
let s = 0; for (let i = 1; i < 10; i++) s += freq[i] // Bass-/Kick-Bereich
level.current = Math.min(1, (s / 9 / 255) * 1.7)
rafRef.current = requestAnimationFrame(tick)
}
tick()
ctxRef.current = ctx; streamRef.current = stream
setDancing(true)
} catch (e) { console.error("Tanz-Audio:", e); stop() }
}, [stop])
const toggleDance = useCallback(() => { dancing ? stop() : void start() }, [dancing, start, stop])
useEffect(() => () => stop(), [stop])
return { danceLevel: level, dancing, toggleDance }
}
client/lucy-desktop/src/renderer/src/styles.css
+9
View File
@@ -176,6 +176,15 @@ body {
.app.overlay .modeToggle { display: none; }
.vadDot { width: 7px; height: 7px; border-radius: 50%; background: #34d399; box-shadow: 0 0 8px #34d399; animation: pulse 1s infinite; }
.danceBtn {
border: 1px solid rgba(167,139,250,0.35); background: rgba(167,139,250,0.12); color: #c4b5fd;
font-size: 11px; padding: 4px 12px; border-radius: 9px; cursor: pointer; -webkit-app-region: no-drag;
transition: background 0.12s, color 0.12s;
}
.danceBtn:hover { background: rgba(167,139,250,0.22); color: #ddd6fe; }
.danceBtn.on { background: rgba(167,139,250,0.3); color: #ede9fe; border-color: rgba(167,139,250,0.6); }
.app.overlay .danceBtn { display: none; } /* im Overlay kein Platz -> nur im Vollfenster */
.side {
width: 320px; flex-shrink: 0; display: flex; flex-direction: column;
border: 1px solid rgba(255,255,255,0.07); border-radius: 14px; background: rgba(255,255,255,0.02);