Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,85 @@
|
||||
// Sequentielle Audio-Wiedergabe für die TTS-Antworten + Pegel-Messung fürs Lippensync.
|
||||
//
|
||||
// Die einzelnen Satz-WAVs kommen nacheinander rein (satzweise Synthese → niedrige Latenz).
|
||||
// Wir spielen sie über EINEN AudioContext geordnet ab und hängen einen AnalyserNode dazwischen,
|
||||
// dessen Energie pro Frame in `level.current` (0..1) landet — der 3D-Avatar liest das im
|
||||
// useFrame und öffnet den Mund entsprechend. Kein Re-Render pro Frame (Mutable-Ref-Muster).
|
||||
|
||||
export class AudioQueue {
|
||||
private ctx: AudioContext
|
||||
private analyser: AnalyserNode
|
||||
private queue: ArrayBuffer[] = []
|
||||
private playing = false
|
||||
private raf = 0
|
||||
private freq: Uint8Array<ArrayBuffer>
|
||||
/** Mutable, vom Avatar pro Frame gelesen. 0 = Mund zu, 1 = weit offen. */
|
||||
readonly level = { current: 0 }
|
||||
onSpeaking?: (speaking: boolean) => void
|
||||
|
||||
constructor() {
|
||||
const Ctor = window.AudioContext || (window as any).webkitAudioContext
|
||||
this.ctx = new Ctor()
|
||||
this.analyser = this.ctx.createAnalyser()
|
||||
this.analyser.fftSize = 256
|
||||
this.analyser.smoothingTimeConstant = 0.6
|
||||
this.analyser.connect(this.ctx.destination)
|
||||
this.freq = new Uint8Array(new ArrayBuffer(this.analyser.frequencyBinCount))
|
||||
}
|
||||
|
||||
async enqueue(buf: ArrayBuffer) {
|
||||
this.queue.push(buf)
|
||||
if (!this.playing) await this.playNext()
|
||||
}
|
||||
|
||||
/** Laufende + wartende Wiedergabe verwerfen (z.B. wenn der Nutzer dazwischenredet). */
|
||||
clear() {
|
||||
this.queue = []
|
||||
}
|
||||
|
||||
private async playNext(): Promise<void> {
|
||||
const buf = this.queue.shift()
|
||||
if (!buf) {
|
||||
this.playing = false
|
||||
this.stopMeter()
|
||||
this.onSpeaking?.(false)
|
||||
return
|
||||
}
|
||||
this.playing = true
|
||||
this.onSpeaking?.(true)
|
||||
if (this.ctx.state === "suspended") {
|
||||
try { await this.ctx.resume() } catch { /* vom User-Gesture freigeschaltet */ }
|
||||
}
|
||||
let audioBuf: AudioBuffer
|
||||
try {
|
||||
audioBuf = await this.ctx.decodeAudioData(buf.slice(0))
|
||||
} catch {
|
||||
return this.playNext() // kaputtes Segment überspringen
|
||||
}
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
src.onended = () => { void this.playNext() }
|
||||
src.start()
|
||||
this.startMeter()
|
||||
}
|
||||
|
||||
private startMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
const tick = () => {
|
||||
this.analyser.getByteFrequencyData(this.freq)
|
||||
// Sprachenergie liegt v.a. in den unteren/mittleren Bändern.
|
||||
const n = Math.min(this.freq.length, 48)
|
||||
let sum = 0
|
||||
for (let i = 2; i < n; i++) sum += this.freq[i]
|
||||
const avg = sum / (n - 2) / 255
|
||||
this.level.current = Math.min(1, avg * 1.9)
|
||||
this.raf = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
private stopMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
this.level.current = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Leichtgewichtige Stimmungs-Heuristik (v1) → treibt die Avatar-Mimik.
|
||||
// Bewusst simpel/regelbasiert (kein Modell): mappt deutschen Antworttext auf eine VRM-Expression.
|
||||
// Spätere Stufe: echte Hermes-Emotion/Audio-Tags. Siehe Plan.
|
||||
|
||||
export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed"
|
||||
|
||||
const RULES: [Emotion, RegExp][] = [
|
||||
["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha|:\)|😊|😄|🎉)/i],
|
||||
["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i],
|
||||
["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i],
|
||||
["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i],
|
||||
]
|
||||
|
||||
export function sentimentToEmotion(text: string): Emotion {
|
||||
for (const [emo, rx] of RULES) if (rx.test(text)) return emo
|
||||
return "neutral"
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Push-to-talk-Aufnahme über MediaRecorder. `start` beim Drücken (Taste/Button), `stop` beim
|
||||
// Loslassen → fertiges Audio-Blob (webm/opus) geht an `onAudio`. Bewusst minimal & generisch.
|
||||
export function usePushToTalk(onAudio: (blob: Blob) => void) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recRef = useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = useRef<Blob[]>([])
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recRef.current) return
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
console.error("Mikrofon-Zugriff verweigert:", e)
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
|
||||
? "audio/webm;codecs=opus"
|
||||
: "audio/webm"
|
||||
const rec = new MediaRecorder(stream, { mimeType: mime })
|
||||
chunksRef.current = []
|
||||
rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) }
|
||||
rec.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: mime })
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current = null
|
||||
recRef.current = null
|
||||
setRecording(false)
|
||||
if (blob.size > 1200) onAudio(blob) // Mini-Blobs (Versehen) ignorieren
|
||||
}
|
||||
rec.start()
|
||||
recRef.current = rec
|
||||
setRecording(true)
|
||||
}, [onAudio])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
recRef.current?.stop()
|
||||
}, [])
|
||||
|
||||
useEffect(() => () => {
|
||||
recRef.current?.stop()
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
}, [])
|
||||
|
||||
return { recording, start, stop }
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { usePushToTalk } from "./usePushToTalk"
|
||||
import { AudioQueue } from "./audio"
|
||||
import { sentimentToEmotion, type Emotion } from "./sentiment"
|
||||
|
||||
// Orchestriert die ganze Voll-Duplex-Schleife im Browser:
|
||||
// PTT-Audio → /api/voice/stt → User-Text
|
||||
// → /api/voice/chat (SSE vom Hermes-Agenten, server-seitiger Verlauf via Session-Id)
|
||||
// → Antwort satzweise schneiden → /api/voice/tts je Satz → AudioQueue (Abspielen + Lippensync)
|
||||
// → Stimmung aus dem Antworttext → Avatar-Mimik
|
||||
//
|
||||
// Avatar-Anbindung ohne Re-Render: `audioLevel` (Mundöffnung) und `emotion` sind Mutable-Refs,
|
||||
// die der 3D-Avatar pro Frame liest.
|
||||
|
||||
export type VoiceStatus = "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error"
|
||||
export interface ChatMsg { role: "user" | "assistant"; text: string }
|
||||
|
||||
const SYSTEM_PROMPT =
|
||||
"Du sprichst per Sprache mit dem Nutzer. Antworte natürlich, freundlich und KNAPP in ganzen, " +
|
||||
"gut vorlesbaren Sätzen. Kein Markdown, keine Codeblöcke, keine Aufzählungszeichen, keine Emojis — " +
|
||||
"reiner Fließtext, den man laut vorlesen kann."
|
||||
|
||||
function getSessionId(): string {
|
||||
let id = localStorage.getItem("mc_voice_session")
|
||||
if (!id) {
|
||||
id = "voice-" + Math.random().toString(36).slice(2) + Date.now().toString(36)
|
||||
localStorage.setItem("mc_voice_session", id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
function readSettings() {
|
||||
return {
|
||||
engine: localStorage.getItem("mc_voice_engine") || "piper",
|
||||
voice: localStorage.getItem("mc_voice_voice") || "",
|
||||
}
|
||||
}
|
||||
|
||||
// Zerlegt einen wachsenden Text-Stream in fertige Sätze. Gibt komplette Sätze zurück und behält
|
||||
// den unvollständigen Rest. So kann das erste TTS schon starten, bevor die Antwort fertig ist.
|
||||
function splitSentences(buffer: string): { sentences: string[]; rest: string } {
|
||||
const sentences: string[] = []
|
||||
const rx = /[^.!?…]+[.!?…]+(\s|$)/g
|
||||
let last = 0
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = rx.exec(buffer))) {
|
||||
sentences.push(m[0].trim())
|
||||
last = rx.lastIndex
|
||||
}
|
||||
return { sentences, rest: buffer.slice(last) }
|
||||
}
|
||||
|
||||
export function useVoiceAgent() {
|
||||
const [status, setStatus] = useState<VoiceStatus>("idle")
|
||||
const [messages, setMessages] = useState<ChatMsg[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const audioLevel = useRef({ current: 0 }) // wird gleich auf die Queue-Pegel gezeigt
|
||||
const emotion = useRef<Emotion>("neutral")
|
||||
const queueRef = useRef<AudioQueue | null>(null)
|
||||
const sessionId = useRef<string>(getSessionId())
|
||||
|
||||
// AudioQueue erst bei Bedarf (nach User-Geste) erzeugen — Autoplay-Policy.
|
||||
const ensureQueue = useCallback(() => {
|
||||
if (!queueRef.current) {
|
||||
const q = new AudioQueue()
|
||||
q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s))
|
||||
queueRef.current = q
|
||||
audioLevel.current = q.level // Avatar liest ab jetzt echte Pegel
|
||||
}
|
||||
return queueRef.current
|
||||
}, [])
|
||||
|
||||
const handleAudio = useCallback(async (blob: Blob) => {
|
||||
setError(null)
|
||||
const queue = ensureQueue()
|
||||
queue.clear() // evtl. laufende Antwort abbrechen (Barge-in)
|
||||
|
||||
// 1) STT
|
||||
setStatus("transcribing")
|
||||
let userText = ""
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", blob, "rec.webm")
|
||||
const r = await fetch("/api/voice/stt", { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`STT ${r.status}`)
|
||||
userText = (await r.json()).text?.trim() || ""
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return
|
||||
}
|
||||
if (!userText) { setStatus("idle"); return }
|
||||
setMessages((m) => [...m, { role: "user", text: userText }])
|
||||
|
||||
// 2) Chat (SSE) → 3) satzweises TTS
|
||||
setStatus("thinking")
|
||||
const { engine, voice } = readSettings()
|
||||
let assistant = ""
|
||||
let pending = ""
|
||||
setMessages((m) => [...m, { role: "assistant", text: "" }])
|
||||
|
||||
const speak = (sentence: string) => {
|
||||
if (!sentence.trim()) return
|
||||
fetch("/api/voice/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: sentence, engine, voice }),
|
||||
})
|
||||
.then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(`TTS ${r.status}`))))
|
||||
.then((buf) => queue.enqueue(buf))
|
||||
.catch((e) => console.error("TTS-Fehler:", e))
|
||||
}
|
||||
|
||||
try {
|
||||
const r = await fetch("/api/voice/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
text: userText,
|
||||
session_id: sessionId.current,
|
||||
system: SYSTEM_PROMPT,
|
||||
}),
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`)
|
||||
|
||||
const reader = r.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let sse = ""
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
sse += dec.decode(value, { stream: true })
|
||||
const events = sse.split("\n\n")
|
||||
sse = events.pop() || ""
|
||||
for (const ev of events) {
|
||||
const line = ev.split("\n").find((l) => l.startsWith("data:"))
|
||||
if (!line) continue
|
||||
const data = line.slice(5).trim()
|
||||
if (data === "[DONE]") continue
|
||||
let json: any
|
||||
try { json = JSON.parse(data) } catch { continue }
|
||||
if (json.error) throw new Error(json.error)
|
||||
const delta = json.choices?.[0]?.delta?.content || ""
|
||||
if (!delta) continue
|
||||
assistant += delta
|
||||
pending += delta
|
||||
emotion.current = sentimentToEmotion(assistant)
|
||||
setMessages((m) => {
|
||||
const copy = m.slice()
|
||||
copy[copy.length - 1] = { role: "assistant", text: assistant }
|
||||
return copy
|
||||
})
|
||||
const { sentences, rest } = splitSentences(pending)
|
||||
pending = rest
|
||||
sentences.forEach(speak)
|
||||
}
|
||||
}
|
||||
if (pending.trim()) speak(pending) // Rest (letzter Satz ohne Satzzeichen)
|
||||
if (!assistant.trim()) setStatus("idle")
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`)
|
||||
}
|
||||
}, [ensureQueue])
|
||||
|
||||
const { recording, start, stop } = usePushToTalk(handleAudio)
|
||||
|
||||
const pressStart = useCallback(() => {
|
||||
ensureQueue()
|
||||
setStatus("listening")
|
||||
void start()
|
||||
}, [ensureQueue, start])
|
||||
|
||||
const pressEnd = useCallback(() => { stop() }, [stop])
|
||||
|
||||
const reset = useCallback(() => {
|
||||
queueRef.current?.clear()
|
||||
setMessages([])
|
||||
setError(null)
|
||||
setStatus("idle")
|
||||
localStorage.removeItem("mc_voice_session")
|
||||
sessionId.current = getSessionId()
|
||||
}, [])
|
||||
|
||||
// Leerlauf-Status zurücksetzen, wenn nichts mehr spricht/aufnimmt.
|
||||
useEffect(() => {
|
||||
if (!recording && (status === "listening")) setStatus("transcribing")
|
||||
}, [recording, status])
|
||||
|
||||
return {
|
||||
status, messages, error, recording,
|
||||
audioLevel, emotion,
|
||||
pressStart, pressEnd, reset,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Winziger IndexedDB-Wrapper, um ein hochgeladenes .vrm (ArrayBuffer) über Reloads hinweg zu
|
||||
// behalten. localStorage scheidet aus (VRMs sind oft 5–40 MB). Ein einziger Record genügt.
|
||||
|
||||
const DB = "mc2-voice"
|
||||
const STORE = "avatar"
|
||||
const KEY = "uploaded-vrm"
|
||||
|
||||
function open(): Promise<IDBDatabase> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = indexedDB.open(DB, 1)
|
||||
req.onupgradeneeded = () => req.result.createObjectStore(STORE)
|
||||
req.onsuccess = () => resolve(req.result)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function saveUploadedVrm(buf: ArrayBuffer): Promise<void> {
|
||||
const db = await open()
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readwrite")
|
||||
tx.objectStore(STORE).put(buf, KEY)
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function loadUploadedVrm(): Promise<ArrayBuffer | null> {
|
||||
const db = await open()
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readonly")
|
||||
const req = tx.objectStore(STORE).get(KEY)
|
||||
req.onsuccess = () => resolve((req.result as ArrayBuffer) ?? null)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user