Frontend: Voice-Umzug in Desktop-App, MC3-Prototyp, UI-Rework-Dateien auf lucy-v2, dist konsistent
- VoiceView + components/voice + lib/voice entfernt (lebt jetzt in client/lucy-desktop)
- mc3/: Basisstation-Prototyp hinter #mc3-Weiche (non-destruktiv)
- roleMeta/useLucyHealth/useExpertMode/ExpertToggle/LucyHealthCard/WarmSetManager (UI-Rework, auf main a341d25 committet, hier nachgezogen)
- dist frisch gebaut (Asset-Stand war inkonsistent: alte geloescht, neue untracked)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,126 @@
|
||||
import {
|
||||
Brain,
|
||||
BrainCog,
|
||||
Code,
|
||||
Eye,
|
||||
BookMarked,
|
||||
Telescope,
|
||||
type LucideIcon,
|
||||
} from "lucide-react"
|
||||
|
||||
// ── EINE Quelle der Wahrheit für ALLE Serving-Rollen ────────────────────────
|
||||
// Klartext statt Kürzel: jede Rolle bekommt einen Zweck-Namen, ein Icon und
|
||||
// eine Ein-Satz-Erklärung, damit die UI ohne Fachjargon verständlich ist.
|
||||
// Das technische Kürzel (`role`) bleibt für den späteren Experten-Modus erhalten.
|
||||
//
|
||||
// Die Rollen hier spiegeln den ECHTEN Stand der Box (/api/models → role):
|
||||
// hermes · heavy · coder · coder-lite · vision · embed · scout
|
||||
// (Kein Phantom-`fast`: das ist nur ein Lane-Alias, keine Modell-Rolle. Lucys
|
||||
// Alltags-Hirn ist die Rolle `hermes` = Qwen3.6.)
|
||||
|
||||
export interface RoleMeta {
|
||||
role: string // technisches Kürzel (Experten-Modus)
|
||||
label: string // Klartext-Zweckname
|
||||
short: string // Kurzform für enge Badges
|
||||
icon: LucideIcon
|
||||
desc: string // Tooltip / Erklärung in einfacher Sprache
|
||||
tone: string // Farbklassen fürs Badge
|
||||
protected?: boolean // Hirn & Gedächtnis: lebenswichtig → nicht löschbar/entladbar (Etappe 3)
|
||||
}
|
||||
|
||||
// Reihenfolge = Anzeige-Reihenfolge in den Slot-Übersichten (Wichtigstes zuerst).
|
||||
export const ROLE_META: RoleMeta[] = [
|
||||
{
|
||||
role: "hermes",
|
||||
label: "Lucys Hirn",
|
||||
short: "Hirn",
|
||||
icon: Brain,
|
||||
desc: "Lucys Alltags-Verstand — denkt und redet mit dir. Immer bereit.",
|
||||
tone: "bg-indigo-500/15 text-indigo-400 border-indigo-500/25",
|
||||
protected: true,
|
||||
},
|
||||
{
|
||||
role: "embed",
|
||||
label: "Lucys Gedächtnis",
|
||||
short: "Gedächtnis",
|
||||
icon: BookMarked,
|
||||
desc: "Merkt sich Wichtiges und findet es später wieder.",
|
||||
tone: "bg-emerald-500/15 text-emerald-400 border-emerald-500/25",
|
||||
protected: true,
|
||||
},
|
||||
{
|
||||
role: "vision",
|
||||
label: "Lucys Augen",
|
||||
short: "Augen",
|
||||
icon: Eye,
|
||||
desc: "Sieht Bilder und Screenshots und beschreibt, was drauf ist.",
|
||||
tone: "bg-pink-500/15 text-pink-400 border-pink-500/25",
|
||||
},
|
||||
{
|
||||
role: "heavy",
|
||||
label: "Großes Hirn",
|
||||
short: "Groß",
|
||||
icon: BrainCog,
|
||||
desc: "Das schwere Geschütz — wird für besonders knifflige Fragen dazugeholt.",
|
||||
tone: "bg-amber-500/15 text-amber-400 border-amber-500/25",
|
||||
},
|
||||
{
|
||||
role: "coder-lite",
|
||||
label: "Programmierer (schnell)",
|
||||
short: "Code schnell",
|
||||
icon: Code,
|
||||
desc: "Schnelles Programmieren im Alltag.",
|
||||
tone: "bg-violet-500/15 text-violet-400 border-violet-500/25",
|
||||
},
|
||||
{
|
||||
role: "coder",
|
||||
label: "Programmierer (gründlich)",
|
||||
short: "Code gründlich",
|
||||
icon: Code,
|
||||
desc: "Schreibt und prüft Code besonders sorgfältig — für schwere Coding-Aufgaben.",
|
||||
tone: "bg-fuchsia-500/15 text-fuchsia-400 border-fuchsia-500/25",
|
||||
},
|
||||
{
|
||||
role: "scout",
|
||||
label: "Späher",
|
||||
short: "Späher",
|
||||
icon: Telescope,
|
||||
desc: "Schneller erster Blick und grobe Einschätzung.",
|
||||
tone: "bg-teal-500/15 text-teal-400 border-teal-500/25",
|
||||
},
|
||||
]
|
||||
|
||||
// Kanonische Rollen-Reihenfolge (ersetzt die alte, veraltete ROLES-Liste).
|
||||
export const ROLES: string[] = ROLE_META.map((m) => m.role)
|
||||
|
||||
const BY_ROLE: Record<string, RoleMeta> = Object.fromEntries(
|
||||
ROLE_META.map((m) => [m.role, m]),
|
||||
)
|
||||
|
||||
// Fallback für unbekannte/neue Rollen — nie „undefined", damit die UID nicht bricht.
|
||||
const UNKNOWN = (role?: string | null): RoleMeta => ({
|
||||
role: role || "?",
|
||||
label: role || "unbekannt",
|
||||
short: role || "?",
|
||||
icon: Brain,
|
||||
desc: "Rolle ohne hinterlegten Klartext-Namen.",
|
||||
tone: "bg-slate-500/15 text-slate-300 border-slate-500/25",
|
||||
})
|
||||
|
||||
// Aliase, die auf eine echte Rolle zeigen (z.B. Lane-Alias `fast` = Hirn).
|
||||
const ALIAS: Record<string, string> = {
|
||||
fast: "hermes",
|
||||
coder_lite: "coder-lite",
|
||||
}
|
||||
|
||||
export function roleMeta(role?: string | null): RoleMeta {
|
||||
if (!role) return UNKNOWN(role)
|
||||
return BY_ROLE[role] || BY_ROLE[ALIAS[role]] || UNKNOWN(role)
|
||||
}
|
||||
|
||||
// Klartext-Name einer Rolle (mit optionaler Kurzform).
|
||||
export const roleLabel = (role?: string | null, short = false) =>
|
||||
short ? roleMeta(role).short : roleMeta(role).label
|
||||
|
||||
// Farbton — bleibt kompatibel zur bisherigen roleTone()-Signatur.
|
||||
export const roleTone = (role?: string | null) => roleMeta(role).tone
|
||||
@@ -0,0 +1,34 @@
|
||||
import { useSyncExternalStore } from "react"
|
||||
|
||||
// Globaler „Einfach/Experte"-Schalter. Einfach (Default) blendet technische
|
||||
// Details (Kontext, Spec, Slots, Ko-Residenz, Lane-Routing …) aus, damit die UI
|
||||
// nicht überfordert. In localStorage gemerkt; alle Consumer aktualisieren via Event.
|
||||
|
||||
const KEY = "mc_expert_mode"
|
||||
const EVT = "mc-expert-mode-change"
|
||||
|
||||
const read = (): boolean => {
|
||||
try {
|
||||
return localStorage.getItem(KEY) === "true"
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function subscribe(cb: () => void) {
|
||||
window.addEventListener(EVT, cb)
|
||||
window.addEventListener("storage", cb)
|
||||
return () => {
|
||||
window.removeEventListener(EVT, cb)
|
||||
window.removeEventListener("storage", cb)
|
||||
}
|
||||
}
|
||||
|
||||
export function setExpertMode(v: boolean) {
|
||||
localStorage.setItem(KEY, String(v))
|
||||
window.dispatchEvent(new Event(EVT))
|
||||
}
|
||||
|
||||
export function useExpertMode(): boolean {
|
||||
return useSyncExternalStore(subscribe, read, () => false)
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
// „Geht's Lucy gut?" — bündelt die vielen technischen Status-Signale zu wenigen
|
||||
// Lucy-Fähigkeiten in Klartext, inkl. Gesamt-Verdikt, Speicher-Ampel und einer
|
||||
// konkreten 1-Klick-Reparatur pro Problem. EINE Quelle für das Status-Panel.
|
||||
|
||||
import { Brain, BookMarked, Plug, Bot, Mic, type LucideIcon } from "lucide-react"
|
||||
import { useHealth, useServices, useModels, useSystemStatus } from "./queries"
|
||||
|
||||
export type CheckStatus = "ok" | "warn" | "down" | "loading"
|
||||
|
||||
// Reparatur-Rezept: entweder einen systemd-Dienst neu starten oder ein Modell laden.
|
||||
// WICHTIG: `service` sind die ECHTEN Unit-Namen (nicht die Anzeige-Namen der
|
||||
// Dienste-Karte), damit /api/maintenance/restart sie akzeptiert.
|
||||
export type Repair =
|
||||
| { kind: "restart"; service: string; label: string; needsSudo?: boolean }
|
||||
| { kind: "loadModel"; model: string; label: string }
|
||||
|
||||
export interface LucyCheck {
|
||||
id: string
|
||||
label: string // Klartext-Name der Fähigkeit
|
||||
icon: LucideIcon
|
||||
status: CheckStatus
|
||||
detail: string // Ein Satz: was ist gerade los
|
||||
critical: boolean // zählt fürs Gesamt-Verdikt „Problem"
|
||||
repair?: Repair
|
||||
}
|
||||
|
||||
export type Verdict = "loading" | "gut" | "warn" | "problem"
|
||||
|
||||
export interface MemoryAmpel {
|
||||
status: "ok" | "warn" | "full" | "unknown"
|
||||
usedGb: number
|
||||
totalGb: number
|
||||
pct: number
|
||||
text: string
|
||||
}
|
||||
|
||||
export interface LucyHealth {
|
||||
verdict: Verdict
|
||||
checks: LucyCheck[]
|
||||
memory: MemoryAmpel
|
||||
problems: LucyCheck[] // kritische Down-Checks (für den Schnellzugriff im Banner)
|
||||
}
|
||||
|
||||
export function useLucyHealth(): LucyHealth {
|
||||
const { data: health } = useHealth()
|
||||
const { data: svc } = useServices()
|
||||
const { data: modelsResp } = useModels()
|
||||
const { data: sys } = useSystemStatus()
|
||||
|
||||
const loaded = !!health && !!svc
|
||||
const svcOk = (frag: string): boolean | undefined =>
|
||||
svc?.services.find((s) => s.name.toLowerCase().includes(frag))?.ok
|
||||
|
||||
const engineOk = health?.engine_reachable
|
||||
const brainReady = health?.brain?.ready
|
||||
const brainName =
|
||||
health?.brain?.model ||
|
||||
modelsResp?.models.find((m) => m.role === "hermes")?.name ||
|
||||
""
|
||||
|
||||
const checks: LucyCheck[] = []
|
||||
|
||||
// 1) Lucys Hirn — die wichtigste Fähigkeit. Engine läuft UND das Hirn-Modell ist geladen?
|
||||
checks.push(((): LucyCheck => {
|
||||
const base = { id: "brain", label: "Lucys Hirn", icon: Brain, critical: true }
|
||||
if (!loaded) return { ...base, status: "loading", detail: "Wird geprüft …" }
|
||||
if (engineOk === false)
|
||||
return {
|
||||
...base,
|
||||
status: "down",
|
||||
detail: "Die Motor-Maschine (Engine) läuft nicht — Lucy kann gerade gar nicht denken.",
|
||||
repair: { kind: "restart", service: "llama-swap", label: "Motor neu starten", needsSudo: true },
|
||||
}
|
||||
if (brainReady === false)
|
||||
return {
|
||||
...base,
|
||||
status: "down",
|
||||
detail: "Lucys Hirn ist gerade eingeschlafen — einmal aufwecken.",
|
||||
repair: brainName
|
||||
? { kind: "loadModel", model: brainName, label: "Hirn aufwecken" }
|
||||
: undefined,
|
||||
}
|
||||
return { ...base, status: "ok", detail: "Wach und ansprechbar." }
|
||||
})())
|
||||
|
||||
// 2) Lucys Gedächtnis — Mem0-Dienst.
|
||||
checks.push(((): LucyCheck => {
|
||||
const base = { id: "memory", label: "Lucys Gedächtnis", icon: BookMarked, critical: true }
|
||||
const ok = svcOk("mem0")
|
||||
if (!loaded || ok === undefined) return { ...base, status: "loading", detail: "Wird geprüft …" }
|
||||
return ok
|
||||
? { ...base, status: "ok", detail: "Merkt sich Neues und findet Altes wieder." }
|
||||
: {
|
||||
...base,
|
||||
status: "down",
|
||||
detail: "Lucy kann sich gerade nichts merken — das Gedächtnis antwortet nicht.",
|
||||
repair: { kind: "restart", service: "mem0-service", label: "Gedächtnis neu starten" },
|
||||
}
|
||||
})())
|
||||
|
||||
// 3) Verbindung (Gateway) — der integrierte OpenAI-Gateway, über den Apps/IDEs Lucy erreichen.
|
||||
checks.push(((): LucyCheck => {
|
||||
const base = { id: "gateway", label: "Verbindung (Gateway)", icon: Plug, critical: true }
|
||||
const ok = health ? health.gateway_reachable : svcOk("integriert")
|
||||
if (!loaded || ok === undefined) return { ...base, status: "loading", detail: "Wird geprüft …" }
|
||||
return ok
|
||||
? { ...base, status: "ok", detail: "Apps und IDEs können Lucy erreichen." }
|
||||
: {
|
||||
...base,
|
||||
status: "down",
|
||||
detail: "Die Verbindungs-Zentrale antwortet nicht — Apps erreichen Lucy nicht.",
|
||||
repair: { kind: "restart", service: "mission-control-2", label: "Zentrale neu starten" },
|
||||
}
|
||||
})())
|
||||
|
||||
// 4) Lucys Agent (Hermes) — der Dienst, der als Lucy antwortet (Terminal/Telegram/Sprechen).
|
||||
checks.push(((): LucyCheck => {
|
||||
const base = { id: "agent", label: "Lucys Agent (Hermes)", icon: Bot, critical: true }
|
||||
const ok = svcOk("hermes-gateway")
|
||||
if (!loaded || ok === undefined) return { ...base, status: "loading", detail: "Wird geprüft …" }
|
||||
return ok
|
||||
? { ...base, status: "ok", detail: "Lucy ist bereit zu reden und zu handeln." }
|
||||
: {
|
||||
...base,
|
||||
status: "down",
|
||||
detail: "Lucys Agent ist offline — sie reagiert gerade nicht.",
|
||||
repair: { kind: "restart", service: "hermes-gateway", label: "Agent neu starten" },
|
||||
}
|
||||
})())
|
||||
|
||||
// 5) Lucys Stimme — Voice-Dienst (STT/TTS). Nicht kritisch: Chat geht auch ohne.
|
||||
checks.push(((): LucyCheck => {
|
||||
const base = { id: "voice", label: "Lucys Stimme", icon: Mic, critical: false }
|
||||
const ok = svcOk("voice")
|
||||
if (!loaded || ok === undefined) return { ...base, status: "loading", detail: "Wird geprüft …" }
|
||||
return ok
|
||||
? { ...base, status: "ok", detail: "Lucy kann hören und sprechen." }
|
||||
: {
|
||||
...base,
|
||||
status: "warn",
|
||||
detail: "Sprechen ist gerade aus — Tippen geht weiter normal.",
|
||||
repair: { kind: "restart", service: "voice-service", label: "Stimme neu starten" },
|
||||
}
|
||||
})())
|
||||
|
||||
// Speicher-Ampel aus dem echten GTT-Pool (unified RAM inkl. KV-Cache).
|
||||
const total = sys?.gpu?.gtt_total || sys?.gpu?.vram_total || 0
|
||||
const used = sys?.gpu?.gtt_used || 0
|
||||
const pct = total > 0 ? Math.min(100, (used / total) * 100) : 0
|
||||
const B = 1024 ** 3
|
||||
const memory: MemoryAmpel = !total
|
||||
? { status: "unknown", usedGb: 0, totalGb: 0, pct: 0, text: "Speicher-Auslastung unbekannt." }
|
||||
: pct >= 90
|
||||
? { status: "full", usedGb: used / B, totalGb: total / B, pct, text: "Speicher fast voll — Lucy könnte langsamer werden." }
|
||||
: pct >= 72
|
||||
? { status: "warn", usedGb: used / B, totalGb: total / B, pct, text: "Speicher gut gefüllt — noch okay." }
|
||||
: { status: "ok", usedGb: used / B, totalGb: total / B, pct, text: "Genug Speicher frei." }
|
||||
|
||||
const anyLoading = checks.some((c) => c.status === "loading")
|
||||
const problems = checks.filter((c) => c.critical && c.status === "down")
|
||||
const warns = checks.filter((c) => c.status === "warn" || (!c.critical && c.status === "down"))
|
||||
const memBad = memory.status === "full"
|
||||
|
||||
const verdict: Verdict = anyLoading && !problems.length
|
||||
? "loading"
|
||||
: problems.length
|
||||
? "problem"
|
||||
: warns.length || memBad
|
||||
? "warn"
|
||||
: "gut"
|
||||
|
||||
return { verdict, checks, memory, problems }
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
// Sequentielle Audio-Wiedergabe für die TTS-Antworten + Pegel-Messung fürs Lippensync.
|
||||
//
|
||||
// Die einzelnen Satz-WAVs kommen nacheinander rein (satzweise Synthese → niedrige Latenz).
|
||||
// Wir spielen sie über EINEN AudioContext geordnet ab und hängen einen AnalyserNode dazwischen,
|
||||
// dessen Energie pro Frame in `level.current` (0..1) landet — der 3D-Avatar liest das im
|
||||
// useFrame und öffnet den Mund entsprechend. Kein Re-Render pro Frame (Mutable-Ref-Muster).
|
||||
|
||||
export class AudioQueue {
|
||||
private ctx: AudioContext
|
||||
private analyser: AnalyserNode
|
||||
private gain: GainNode
|
||||
private queue: ArrayBuffer[] = []
|
||||
private playing = false
|
||||
private raf = 0
|
||||
private freq: Uint8Array<ArrayBuffer>
|
||||
/** Mutable, vom Avatar pro Frame gelesen. 0 = Mund zu, 1 = weit offen. */
|
||||
readonly level = { current: 0 }
|
||||
onSpeaking?: (speaking: boolean) => void
|
||||
|
||||
/** Lautstärke 0..1.5 aus localStorage (Default 0.6 — die Stimmen waren zu laut). */
|
||||
static readVolume(): number {
|
||||
const raw = localStorage.getItem("mc_voice_volume")
|
||||
if (raw === null || raw === "") return 0.6 // Number(null)===0 → sonst aus Versehen stumm
|
||||
const v = Number(raw)
|
||||
return Number.isNaN(v) ? 0.6 : Math.max(0, Math.min(1.5, v))
|
||||
}
|
||||
|
||||
constructor() {
|
||||
const Ctor = window.AudioContext || (window as any).webkitAudioContext
|
||||
this.ctx = new Ctor()
|
||||
this.analyser = this.ctx.createAnalyser()
|
||||
this.analyser.fftSize = 256
|
||||
this.analyser.smoothingTimeConstant = 0.6
|
||||
// Kette: Quelle → Analyser (Lippensync liest vollen Pegel) → Gain (Lautstärke) → Ausgang.
|
||||
this.gain = this.ctx.createGain()
|
||||
this.gain.gain.value = AudioQueue.readVolume()
|
||||
this.analyser.connect(this.gain)
|
||||
this.gain.connect(this.ctx.destination)
|
||||
this.freq = new Uint8Array(new ArrayBuffer(this.analyser.frequencyBinCount))
|
||||
// Live-Lautstärke vom Slider.
|
||||
window.addEventListener("mc-voice-volume", (e: Event) => {
|
||||
const v = Number((e as CustomEvent).detail)
|
||||
if (!Number.isNaN(v)) this.gain.gain.value = Math.max(0, Math.min(1.5, v))
|
||||
})
|
||||
}
|
||||
|
||||
async enqueue(buf: ArrayBuffer) {
|
||||
this.queue.push(buf)
|
||||
if (!this.playing) await this.playNext()
|
||||
}
|
||||
|
||||
/** Laufende + wartende Wiedergabe verwerfen (z.B. wenn der Nutzer dazwischenredet). */
|
||||
clear() {
|
||||
this.queue = []
|
||||
}
|
||||
|
||||
private async playNext(): Promise<void> {
|
||||
const buf = this.queue.shift()
|
||||
if (!buf) {
|
||||
this.playing = false
|
||||
this.stopMeter()
|
||||
this.onSpeaking?.(false)
|
||||
return
|
||||
}
|
||||
this.playing = true
|
||||
this.onSpeaking?.(true)
|
||||
if (this.ctx.state === "suspended") {
|
||||
try { await this.ctx.resume() } catch { /* vom User-Gesture freigeschaltet */ }
|
||||
}
|
||||
let audioBuf: AudioBuffer
|
||||
try {
|
||||
audioBuf = await this.ctx.decodeAudioData(buf.slice(0))
|
||||
} catch {
|
||||
return this.playNext() // kaputtes Segment überspringen
|
||||
}
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
src.onended = () => { void this.playNext() }
|
||||
src.start()
|
||||
this.startMeter()
|
||||
}
|
||||
|
||||
private startMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
const tick = () => {
|
||||
this.analyser.getByteFrequencyData(this.freq)
|
||||
// Sprachenergie liegt v.a. in den unteren/mittleren Bändern.
|
||||
const n = Math.min(this.freq.length, 48)
|
||||
let sum = 0
|
||||
for (let i = 2; i < n; i++) sum += this.freq[i]
|
||||
const avg = sum / (n - 2) / 255
|
||||
this.level.current = Math.min(1, avg * 1.9)
|
||||
this.raf = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
private stopMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
this.level.current = 0
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
// Leichtgewichtige Stimmungs-Heuristik (v1) → treibt die Avatar-Mimik.
|
||||
// Bewusst simpel/regelbasiert (kein Modell): mappt deutschen Antworttext auf eine VRM-Expression.
|
||||
// Spätere Stufe: echte Hermes-Emotion/Audio-Tags. Siehe Plan.
|
||||
|
||||
export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed"
|
||||
|
||||
const RULES: [Emotion, RegExp][] = [
|
||||
["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha|:\)|😊|😄|🎉)/i],
|
||||
["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i],
|
||||
["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i],
|
||||
["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i],
|
||||
]
|
||||
|
||||
export function sentimentToEmotion(text: string): Emotion {
|
||||
for (const [emo, rx] of RULES) if (rx.test(text)) return emo
|
||||
return "neutral"
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Push-to-talk-Aufnahme über MediaRecorder. `start` beim Drücken (Taste/Button), `stop` beim
|
||||
// Loslassen → fertiges Audio-Blob (webm/opus) geht an `onAudio`. Bewusst minimal & generisch.
|
||||
export function usePushToTalk(onAudio: (blob: Blob) => void) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recRef = useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = useRef<Blob[]>([])
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recRef.current) return
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
console.error("Mikrofon-Zugriff verweigert:", e)
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
|
||||
? "audio/webm;codecs=opus"
|
||||
: "audio/webm"
|
||||
const rec = new MediaRecorder(stream, { mimeType: mime })
|
||||
chunksRef.current = []
|
||||
rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) }
|
||||
rec.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: mime })
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current = null
|
||||
recRef.current = null
|
||||
setRecording(false)
|
||||
if (blob.size > 1200) onAudio(blob) // Mini-Blobs (Versehen) ignorieren
|
||||
}
|
||||
rec.start()
|
||||
recRef.current = rec
|
||||
setRecording(true)
|
||||
}, [onAudio])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
recRef.current?.stop()
|
||||
}, [])
|
||||
|
||||
useEffect(() => () => {
|
||||
recRef.current?.stop()
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
}, [])
|
||||
|
||||
return { recording, start, stop }
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { usePushToTalk } from "./usePushToTalk"
|
||||
import { AudioQueue } from "./audio"
|
||||
import { sentimentToEmotion, type Emotion } from "./sentiment"
|
||||
|
||||
// Orchestriert die ganze Voll-Duplex-Schleife im Browser:
|
||||
// PTT-Audio → /api/voice/stt → User-Text
|
||||
// → /api/voice/chat (SSE vom Hermes-Agenten, server-seitiger Verlauf via Session-Id)
|
||||
// → Antwort satzweise schneiden → /api/voice/tts je Satz → AudioQueue (Abspielen + Lippensync)
|
||||
// → Stimmung aus dem Antworttext → Avatar-Mimik
|
||||
//
|
||||
// Avatar-Anbindung ohne Re-Render: `audioLevel` (Mundöffnung) und `emotion` sind Mutable-Refs,
|
||||
// die der 3D-Avatar pro Frame liest.
|
||||
|
||||
export type VoiceStatus = "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error"
|
||||
export interface ChatMsg { role: "user" | "assistant"; text: string }
|
||||
|
||||
const SYSTEM_PROMPT =
|
||||
"Du sprichst per Sprache mit dem Nutzer. Antworte natürlich, freundlich und KNAPP in ganzen, " +
|
||||
"gut vorlesbaren Sätzen. Kein Markdown, keine Codeblöcke, keine Aufzählungszeichen, keine Emojis — " +
|
||||
"reiner Fließtext, den man laut vorlesen kann. Verwende IMMER echte deutsche Umlaute (ä, ö, ü, ß) " +
|
||||
"und NIEMALS Umschreibungen wie ae, oe, ue oder ss. Nenne möglichst keine langen URLs oder Codebefehle."
|
||||
|
||||
// Entfernt, was nicht vorgelesen werden soll (Links, Markdown, Code, Emojis), bevor der Satz ans TTS geht.
|
||||
// Die Anzeige im Chat bleibt unangetastet — nur die gesprochene Fassung wird gesäubert.
|
||||
function cleanForTTS(s: string): string {
|
||||
return s
|
||||
.replace(/```[\s\S]*?```/g, " ") // Codeblöcke
|
||||
.replace(/`([^`]*)`/g, "$1") // Inline-Code
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // [Text](url) → Text
|
||||
.replace(/https?:\/\/\S+/gi, " ") // URLs
|
||||
.replace(/www\.\S+/gi, " ")
|
||||
.replace(/\b\S+@\S+\.\S+\b/g, " ") // E-Mails
|
||||
.replace(/[*_#>~|`]+/g, " ") // Markdown-Zeichen
|
||||
.replace(/^\s*[-•·]\s+/gm, " ") // Listen-Bullets
|
||||
// Sonderzeichen aussprechbar machen (sonst liest die Stimme sie wörtlich/komisch):
|
||||
.replace(/\s*&\s*/g, " und ")
|
||||
.replace(/(\d)\s*%/g, "$1 Prozent").replace(/%/g, " Prozent ")
|
||||
.replace(/(\d)\s*°\s*C?/g, "$1 Grad").replace(/°/g, " Grad ")
|
||||
.replace(/\s*=\s*/g, " gleich ")
|
||||
.replace(/\s*\/\s*/g, " ") // Schrägstriche → Pause statt „Schrägstrich"
|
||||
.replace(/[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}]/gu, "") // Emojis/Symbole
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
}
|
||||
|
||||
function getSessionId(): string {
|
||||
let id = localStorage.getItem("mc_voice_session")
|
||||
if (!id) {
|
||||
id = "voice-" + Math.random().toString(36).slice(2) + Date.now().toString(36)
|
||||
localStorage.setItem("mc_voice_session", id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
function readSettings() {
|
||||
return {
|
||||
engine: localStorage.getItem("mc_voice_engine") || "elevenlabs",
|
||||
voice: localStorage.getItem("mc_voice_voice") || "",
|
||||
}
|
||||
}
|
||||
|
||||
// Zerlegt einen wachsenden Text-Stream in fertige Sätze. Gibt komplette Sätze zurück und behält
|
||||
// den unvollständigen Rest. So kann das erste TTS schon starten, bevor die Antwort fertig ist.
|
||||
function splitSentences(buffer: string): { sentences: string[]; rest: string } {
|
||||
const sentences: string[] = []
|
||||
const rx = /[^.!?…]+[.!?…]+(\s|$)/g
|
||||
let last = 0
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = rx.exec(buffer))) {
|
||||
sentences.push(m[0].trim())
|
||||
last = rx.lastIndex
|
||||
}
|
||||
return { sentences, rest: buffer.slice(last) }
|
||||
}
|
||||
|
||||
export function useVoiceAgent() {
|
||||
const [status, setStatus] = useState<VoiceStatus>("idle")
|
||||
const [messages, setMessages] = useState<ChatMsg[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const audioLevel = useRef({ current: 0 }) // wird gleich auf die Queue-Pegel gezeigt
|
||||
const emotion = useRef<Emotion>("neutral")
|
||||
const queueRef = useRef<AudioQueue | null>(null)
|
||||
const sessionId = useRef<string>(getSessionId())
|
||||
|
||||
// AudioQueue erst bei Bedarf (nach User-Geste) erzeugen — Autoplay-Policy.
|
||||
const ensureQueue = useCallback(() => {
|
||||
if (!queueRef.current) {
|
||||
const q = new AudioQueue()
|
||||
q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s))
|
||||
queueRef.current = q
|
||||
audioLevel.current = q.level // Avatar liest ab jetzt echte Pegel
|
||||
}
|
||||
return queueRef.current
|
||||
}, [])
|
||||
|
||||
const handleAudio = useCallback(async (blob: Blob) => {
|
||||
setError(null)
|
||||
const queue = ensureQueue()
|
||||
queue.clear() // evtl. laufende Antwort abbrechen (Barge-in)
|
||||
|
||||
// 1) STT
|
||||
setStatus("transcribing")
|
||||
let userText = ""
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", blob, "rec.webm")
|
||||
const r = await fetch("/api/voice/stt", { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`STT ${r.status}`)
|
||||
userText = (await r.json()).text?.trim() || ""
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return
|
||||
}
|
||||
if (!userText) { setStatus("idle"); return }
|
||||
setMessages((m) => [...m, { role: "user", text: userText }])
|
||||
|
||||
// 2) Chat (SSE) → 3) satzweises TTS
|
||||
setStatus("thinking")
|
||||
const { engine, voice } = readSettings()
|
||||
let assistant = ""
|
||||
let pending = ""
|
||||
setMessages((m) => [...m, { role: "assistant", text: "" }])
|
||||
|
||||
// TTS SERIELL abarbeiten (Satz für Satz), nicht parallel: Chatterbox ist nicht thread-safe
|
||||
// (parallele Generierungen → 502) und Cloud-Engines mögen keine gleichzeitigen Calls. Hält
|
||||
// außerdem die Reihenfolge. Die Audio-Wiedergabe selbst puffert die AudioQueue.
|
||||
let ttsChain: Promise<void> = Promise.resolve()
|
||||
let spokeAny = false
|
||||
let ttsFailed = false
|
||||
const speak = (sentence: string) => {
|
||||
const spoken = cleanForTTS(sentence)
|
||||
if (!spoken) return // z.B. ein Satz, der nur aus einer URL bestand
|
||||
ttsChain = ttsChain.then(async () => {
|
||||
try {
|
||||
const r = await fetch("/api/voice/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: spoken, engine, voice }),
|
||||
})
|
||||
if (!r.ok) throw new Error(`TTS ${r.status}`)
|
||||
await queue.enqueue(await r.arrayBuffer())
|
||||
spokeAny = true
|
||||
} catch (e) {
|
||||
ttsFailed = true
|
||||
console.error("TTS-Fehler:", e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const r = await fetch("/api/voice/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
text: userText,
|
||||
session_id: sessionId.current,
|
||||
system: SYSTEM_PROMPT,
|
||||
}),
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`)
|
||||
|
||||
const reader = r.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let sse = ""
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
sse += dec.decode(value, { stream: true })
|
||||
const events = sse.split("\n\n")
|
||||
sse = events.pop() || ""
|
||||
for (const ev of events) {
|
||||
const line = ev.split("\n").find((l) => l.startsWith("data:"))
|
||||
if (!line) continue
|
||||
const data = line.slice(5).trim()
|
||||
if (data === "[DONE]") continue
|
||||
let json: any
|
||||
try { json = JSON.parse(data) } catch { continue }
|
||||
if (json.error) throw new Error(json.error)
|
||||
const delta = json.choices?.[0]?.delta?.content || ""
|
||||
if (!delta) continue
|
||||
assistant += delta
|
||||
pending += delta
|
||||
emotion.current = sentimentToEmotion(assistant)
|
||||
setMessages((m) => {
|
||||
const copy = m.slice()
|
||||
copy[copy.length - 1] = { role: "assistant", text: assistant }
|
||||
return copy
|
||||
})
|
||||
const { sentences, rest } = splitSentences(pending)
|
||||
pending = rest
|
||||
sentences.forEach(speak)
|
||||
}
|
||||
}
|
||||
if (pending.trim()) speak(pending) // Rest (letzter Satz ohne Satzzeichen)
|
||||
await ttsChain // auf alle (seriellen) TTS-Calls warten
|
||||
if (!assistant.trim()) { setStatus("idle"); return }
|
||||
// Nichts abgespielt → nicht ewig bei „denkt" hängen, Fehler zeigen.
|
||||
if (!spokeAny) {
|
||||
setStatus("error")
|
||||
setError(ttsFailed ? "Sprachausgabe fehlgeschlagen — Engine/Stimme prüfen." : "Keine Sprachausgabe erzeugt.")
|
||||
}
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`)
|
||||
}
|
||||
}, [ensureQueue])
|
||||
|
||||
const { recording, start, stop } = usePushToTalk(handleAudio)
|
||||
|
||||
const pressStart = useCallback(() => {
|
||||
ensureQueue()
|
||||
setStatus("listening")
|
||||
void start()
|
||||
}, [ensureQueue, start])
|
||||
|
||||
const pressEnd = useCallback(() => { stop() }, [stop])
|
||||
|
||||
const reset = useCallback(() => {
|
||||
queueRef.current?.clear()
|
||||
setMessages([])
|
||||
setError(null)
|
||||
setStatus("idle")
|
||||
localStorage.removeItem("mc_voice_session")
|
||||
sessionId.current = getSessionId()
|
||||
}, [])
|
||||
|
||||
// Leerlauf-Status zurücksetzen, wenn nichts mehr spricht/aufnimmt.
|
||||
useEffect(() => {
|
||||
if (!recording && (status === "listening")) setStatus("transcribing")
|
||||
}, [recording, status])
|
||||
|
||||
return {
|
||||
status, messages, error, recording,
|
||||
audioLevel, emotion,
|
||||
pressStart, pressEnd, reset,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user