Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -8,6 +8,7 @@ import { ConnectView } from "@/views/ConnectView"
|
||||
import { MemoryView } from "@/views/MemoryView"
|
||||
import { AgentView } from "@/views/AgentView"
|
||||
import { TerminalView } from "@/views/TerminalView"
|
||||
import { VoiceView } from "@/views/VoiceView"
|
||||
import { GuideView } from "@/views/GuideView"
|
||||
import { Placeholder } from "@/views/Placeholder"
|
||||
import { SystemDrawer } from "@/components/SystemDrawer"
|
||||
@@ -191,8 +192,9 @@ export default function App() {
|
||||
{view === "memory" && <MemoryView />}
|
||||
{view === "agent" && <AgentView />}
|
||||
{view === "terminal" && <TerminalView />}
|
||||
{view === "voice" && <VoiceView />}
|
||||
{view === "guide" && <GuideView />}
|
||||
{!["dashboard", "models", "connect", "memory", "agent", "terminal", "guide"].includes(view) && (
|
||||
{!["dashboard", "models", "connect", "memory", "agent", "terminal", "voice", "guide"].includes(view) && (
|
||||
<Placeholder title={active.label} hint={active.hint} />
|
||||
)}
|
||||
</main>
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { Canvas, useFrame } from "@react-three/fiber"
|
||||
import { OrbitControls } from "@react-three/drei"
|
||||
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
||||
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
||||
import type { Emotion } from "@/lib/voice/sentiment"
|
||||
|
||||
// 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und
|
||||
// stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame).
|
||||
|
||||
type LevelRef = MutableRefObject<{ current: number }> // audioLevel.current.current = Pegel 0..1
|
||||
type EmotionRef = MutableRefObject<Emotion>
|
||||
|
||||
// VRMs laden in T-Pose (Bindepose, Arme waagerecht). Wir senken Ober-/Unterarme auf den
|
||||
// normalisierten Humanoid-Knoten zu einer ruhigen A-Pose ab — sieht sofort natürlich aus.
|
||||
// (Echte Idle-Animation wäre die spätere Stufe.)
|
||||
function applyRestPose(vrm: VRM) {
|
||||
const set = (name: any, x: number, y: number, z: number) => {
|
||||
const b = vrm.humanoid?.getNormalizedBoneNode(name)
|
||||
if (b) b.rotation.set(x, y, z)
|
||||
}
|
||||
set("leftUpperArm", 0, 0, 1.2) // Arm runter an die Seite
|
||||
set("rightUpperArm", 0, 0, -1.2)
|
||||
set("leftLowerArm", 0, -0.2, 0) // leichte Beugung
|
||||
set("rightLowerArm", 0, 0.2, 0)
|
||||
vrm.humanoid?.update()
|
||||
}
|
||||
|
||||
const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
|
||||
const EMO_TO_EXPR: Record<Emotion, string | null> = {
|
||||
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
|
||||
}
|
||||
|
||||
function VrmModel({ url, audioLevel, emotion, onError }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void
|
||||
}) {
|
||||
const [vrm, setVrm] = useState<VRM | null>(null)
|
||||
const smooth = useRef<Record<string, number>>({})
|
||||
const blink = useRef({ t: 0, next: 3, active: 0 })
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false
|
||||
let loaded: VRM | null = null
|
||||
const loader = new GLTFLoader()
|
||||
loader.register((parser) => new VRMLoaderPlugin(parser))
|
||||
loader.load(
|
||||
url,
|
||||
(gltf) => {
|
||||
if (disposed) return
|
||||
const v = gltf.userData.vrm as VRM | undefined
|
||||
if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return }
|
||||
VRMUtils.removeUnnecessaryVertices(gltf.scene)
|
||||
if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
|
||||
v.scene.rotation.y = Math.PI // dem Betrachter zuwenden
|
||||
applyRestPose(v) // T-Pose → entspannte A-Pose (Arme unten)
|
||||
loaded = v
|
||||
setVrm(v)
|
||||
},
|
||||
undefined,
|
||||
(err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") },
|
||||
)
|
||||
return () => {
|
||||
disposed = true
|
||||
if (loaded) VRMUtils.deepDispose(loaded.scene)
|
||||
setVrm(null)
|
||||
}
|
||||
}, [url, onError])
|
||||
|
||||
useFrame((_, delta) => {
|
||||
if (!vrm) return
|
||||
const em = vrm.expressionManager
|
||||
if (em) {
|
||||
// Lippensync: 'aa' folgt geglättet dem Audiopegel.
|
||||
const target = audioLevel.current?.current ?? 0
|
||||
const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6
|
||||
smooth.current.aa = aa
|
||||
em.setValue("aa", aa)
|
||||
|
||||
// Mimik: weich zur Ziel-Expression lerpen.
|
||||
const want = EMO_TO_EXPR[emotion.current]
|
||||
for (const name of EXPRESSIONS) {
|
||||
const tv = want === name ? 0.75 : 0
|
||||
const cv = smooth.current[name] ?? 0
|
||||
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
||||
smooth.current[name] = nv
|
||||
em.setValue(name, nv)
|
||||
}
|
||||
|
||||
// Blinzeln: kurzer Dreieckspuls alle 3–7 s.
|
||||
const b = blink.current
|
||||
b.t += delta
|
||||
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
||||
let blinkVal = 0
|
||||
if (b.active > 0) {
|
||||
b.active -= delta
|
||||
const p = 1 - b.active / 0.16 // 0..1 Fortschritt
|
||||
blinkVal = 1 - Math.abs(p - 0.5) * 2 // 0 → 1 → 0
|
||||
}
|
||||
em.setValue("blink", Math.max(0, blinkVal))
|
||||
}
|
||||
vrm.update(delta)
|
||||
})
|
||||
|
||||
return vrm ? <primitive object={vrm.scene} /> : null
|
||||
}
|
||||
|
||||
export function Avatar3D({ url, audioLevel, emotion }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef
|
||||
}) {
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
return (
|
||||
<div className="relative h-full w-full">
|
||||
<Canvas
|
||||
camera={{ position: [0, 1.35, 1.25], fov: 30 }}
|
||||
gl={{ alpha: true, antialias: true }}
|
||||
style={{ background: "transparent" }}
|
||||
>
|
||||
<ambientLight intensity={0.85} />
|
||||
<directionalLight position={[1, 2, 2]} intensity={1.1} />
|
||||
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
|
||||
{/* key=url → bei Avatarwechsel sauber neu mounten */}
|
||||
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} onError={setErr} />
|
||||
<OrbitControls
|
||||
target={[0, 1.3, 0]}
|
||||
enablePan={false}
|
||||
minDistance={0.7}
|
||||
maxDistance={3}
|
||||
minPolarAngle={Math.PI / 3}
|
||||
maxPolarAngle={Math.PI / 1.8}
|
||||
/>
|
||||
</Canvas>
|
||||
{err && (
|
||||
<div className="absolute inset-x-0 bottom-3 mx-auto w-fit rounded-md bg-red-500/15 border border-red-500/30 px-3 py-1.5 text-xs text-red-300">
|
||||
{err}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { Upload, Link2, ExternalLink, Check, Sparkles } from "lucide-react"
|
||||
import { saveUploadedVrm, loadUploadedVrm } from "@/lib/voice/vrmStore"
|
||||
|
||||
// Avatar selbst aussuchen: kuratierte Galerie (öffentliche, CORS-freie VRMs) + eigenes .vrm
|
||||
// hochladen + per URL laden + VRoid-Hub-Link. Dazu die Stimm-Auswahl (Engine + Stimme).
|
||||
// Auswahl bleibt in localStorage / IndexedDB erhalten.
|
||||
|
||||
interface GalleryItem { id: string; label: string; url: string; note?: string }
|
||||
|
||||
// Verifiziert: 200 + Access-Control-Allow-Origin:* (im Browser ladbar).
|
||||
const GALLERY: GalleryItem[] = [
|
||||
{ id: "sample-a", label: "VRoid Sample A", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_A.vrm", note: "Anime, weiblich" },
|
||||
{ id: "sample-b", label: "VRoid Sample B", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_B.vrm", note: "Anime, männlich" },
|
||||
{ id: "pixiv", label: "Pixiv Demo", url: "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm", note: "VRM1-Testmodell" },
|
||||
]
|
||||
|
||||
export const DEFAULT_AVATAR = GALLERY[0].url
|
||||
|
||||
interface Voice { engine: string; id: string; label: string; clonable?: boolean }
|
||||
|
||||
export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
avatarUrl: string; onAvatarChange: (url: string) => void
|
||||
}) {
|
||||
const [urlInput, setUrlInput] = useState("")
|
||||
const [voices, setVoices] = useState<Voice[]>([])
|
||||
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
|
||||
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
|
||||
const fileRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
// Hochgeladenes VRM nach Reload wiederherstellen.
|
||||
useEffect(() => {
|
||||
if (localStorage.getItem("mc_voice_avatar_uploaded") === "1") {
|
||||
loadUploadedVrm().then((buf) => {
|
||||
if (buf) onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
})
|
||||
}
|
||||
}, [onAvatarChange])
|
||||
|
||||
// Stimmen vom Sidecar holen.
|
||||
useEffect(() => {
|
||||
fetch("/api/voice/voices")
|
||||
.then((r) => (r.ok ? r.json() : Promise.reject()))
|
||||
.then((d) => setVoices(d.voices || []))
|
||||
.catch(() => setVoices([]))
|
||||
}, [])
|
||||
|
||||
const pickGallery = (url: string) => {
|
||||
localStorage.setItem("mc_voice_avatar_url", url)
|
||||
localStorage.removeItem("mc_voice_avatar_uploaded")
|
||||
onAvatarChange(url)
|
||||
}
|
||||
|
||||
const onUpload = async (file: File) => {
|
||||
const buf = await file.arrayBuffer()
|
||||
await saveUploadedVrm(buf)
|
||||
localStorage.setItem("mc_voice_avatar_uploaded", "1")
|
||||
localStorage.removeItem("mc_voice_avatar_url")
|
||||
onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
}
|
||||
|
||||
const loadUrl = () => {
|
||||
const u = urlInput.trim()
|
||||
if (u) pickGallery(u)
|
||||
}
|
||||
|
||||
const saveVoice = (eng: string, v: string) => {
|
||||
setEngine(eng); setVoice(v)
|
||||
localStorage.setItem("mc_voice_engine", eng)
|
||||
localStorage.setItem("mc_voice_voice", v)
|
||||
}
|
||||
|
||||
const enginesAvail = Array.from(new Set(voices.map((v) => v.engine)))
|
||||
const voicesForEngine = voices.filter((v) => v.engine === engine)
|
||||
|
||||
return (
|
||||
<div className="space-y-5 text-sm">
|
||||
{/* Galerie */}
|
||||
<div>
|
||||
<div className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">Avatar</div>
|
||||
<div className="space-y-1.5">
|
||||
{GALLERY.map((g) => {
|
||||
const active = avatarUrl === g.url
|
||||
return (
|
||||
<button
|
||||
key={g.id}
|
||||
onClick={() => pickGallery(g.url)}
|
||||
className={`flex w-full items-center justify-between rounded-md border px-3 py-2 text-left transition-colors ${
|
||||
active ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
<span>
|
||||
<span className="font-medium">{g.label}</span>
|
||||
{g.note && <span className="ml-2 text-[11px] text-muted-foreground">{g.note}</span>}
|
||||
</span>
|
||||
{active && <Check className="h-4 w-4" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Eigenes Modell */}
|
||||
<div className="space-y-2">
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept=".vrm,model/gltf-binary"
|
||||
className="hidden"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) void onUpload(f) }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => fileRef.current?.click()}
|
||||
className="flex w-full items-center gap-2 rounded-md border border-border/40 bg-background/40 px-3 py-2 hover:bg-accent transition-colors"
|
||||
>
|
||||
<Upload className="h-4 w-4" /> Eigenes .vrm hochladen
|
||||
</button>
|
||||
<div className="flex gap-1.5">
|
||||
<input
|
||||
value={urlInput}
|
||||
onChange={(e) => setUrlInput(e.target.value)}
|
||||
onKeyDown={(e) => e.key === "Enter" && loadUrl()}
|
||||
placeholder="…oder VRM-URL einfügen"
|
||||
className="min-w-0 flex-1 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
/>
|
||||
<button onClick={loadUrl} className="rounded-md border border-border/40 bg-background/40 px-2.5 hover:bg-accent" title="Laden">
|
||||
<Link2 className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
<a
|
||||
href="https://hub.vroid.com/en/characters"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
className="flex items-center gap-1.5 text-xs text-primary/80 hover:text-primary"
|
||||
>
|
||||
<ExternalLink className="h-3.5 w-3.5" /> Mehr Avatare auf VRoid Hub (kostenlos) → herunterladen & hochladen
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{/* Stimme */}
|
||||
<div>
|
||||
<div className="mb-2 flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||
<Sparkles className="h-3.5 w-3.5" /> Stimme
|
||||
</div>
|
||||
{voices.length === 0 ? (
|
||||
<div className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-300">
|
||||
Voice-Dienst nicht erreichbar — Stimmen werden geladen, sobald der Sidecar läuft.
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="flex gap-1.5">
|
||||
{enginesAvail.map((eng) => (
|
||||
<button
|
||||
key={eng}
|
||||
onClick={() => saveVoice(eng, "")}
|
||||
className={`flex-1 rounded-md border px-2.5 py-1.5 text-xs capitalize transition-colors ${
|
||||
engine === eng ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
{eng === "piper" ? "Piper (schnell)" : eng === "chatterbox" ? "Chatterbox (premium)" : eng}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<select
|
||||
value={voice}
|
||||
onChange={(e) => saveVoice(engine, e.target.value)}
|
||||
className="w-full rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
>
|
||||
<option value="">Standardstimme</option>
|
||||
{voicesForEngine.map((v) => (
|
||||
<option key={v.id} value={v.id}>{v.label}{v.clonable ? " · klonbar" : ""}</option>
|
||||
))}
|
||||
</select>
|
||||
{engine === "chatterbox" && (
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Natürlichste Stimme + Voice-Cloning.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Sequentielle Audio-Wiedergabe für die TTS-Antworten + Pegel-Messung fürs Lippensync.
|
||||
//
|
||||
// Die einzelnen Satz-WAVs kommen nacheinander rein (satzweise Synthese → niedrige Latenz).
|
||||
// Wir spielen sie über EINEN AudioContext geordnet ab und hängen einen AnalyserNode dazwischen,
|
||||
// dessen Energie pro Frame in `level.current` (0..1) landet — der 3D-Avatar liest das im
|
||||
// useFrame und öffnet den Mund entsprechend. Kein Re-Render pro Frame (Mutable-Ref-Muster).
|
||||
|
||||
export class AudioQueue {
|
||||
private ctx: AudioContext
|
||||
private analyser: AnalyserNode
|
||||
private queue: ArrayBuffer[] = []
|
||||
private playing = false
|
||||
private raf = 0
|
||||
private freq: Uint8Array<ArrayBuffer>
|
||||
/** Mutable, vom Avatar pro Frame gelesen. 0 = Mund zu, 1 = weit offen. */
|
||||
readonly level = { current: 0 }
|
||||
onSpeaking?: (speaking: boolean) => void
|
||||
|
||||
constructor() {
|
||||
const Ctor = window.AudioContext || (window as any).webkitAudioContext
|
||||
this.ctx = new Ctor()
|
||||
this.analyser = this.ctx.createAnalyser()
|
||||
this.analyser.fftSize = 256
|
||||
this.analyser.smoothingTimeConstant = 0.6
|
||||
this.analyser.connect(this.ctx.destination)
|
||||
this.freq = new Uint8Array(new ArrayBuffer(this.analyser.frequencyBinCount))
|
||||
}
|
||||
|
||||
async enqueue(buf: ArrayBuffer) {
|
||||
this.queue.push(buf)
|
||||
if (!this.playing) await this.playNext()
|
||||
}
|
||||
|
||||
/** Laufende + wartende Wiedergabe verwerfen (z.B. wenn der Nutzer dazwischenredet). */
|
||||
clear() {
|
||||
this.queue = []
|
||||
}
|
||||
|
||||
private async playNext(): Promise<void> {
|
||||
const buf = this.queue.shift()
|
||||
if (!buf) {
|
||||
this.playing = false
|
||||
this.stopMeter()
|
||||
this.onSpeaking?.(false)
|
||||
return
|
||||
}
|
||||
this.playing = true
|
||||
this.onSpeaking?.(true)
|
||||
if (this.ctx.state === "suspended") {
|
||||
try { await this.ctx.resume() } catch { /* vom User-Gesture freigeschaltet */ }
|
||||
}
|
||||
let audioBuf: AudioBuffer
|
||||
try {
|
||||
audioBuf = await this.ctx.decodeAudioData(buf.slice(0))
|
||||
} catch {
|
||||
return this.playNext() // kaputtes Segment überspringen
|
||||
}
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
src.onended = () => { void this.playNext() }
|
||||
src.start()
|
||||
this.startMeter()
|
||||
}
|
||||
|
||||
private startMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
const tick = () => {
|
||||
this.analyser.getByteFrequencyData(this.freq)
|
||||
// Sprachenergie liegt v.a. in den unteren/mittleren Bändern.
|
||||
const n = Math.min(this.freq.length, 48)
|
||||
let sum = 0
|
||||
for (let i = 2; i < n; i++) sum += this.freq[i]
|
||||
const avg = sum / (n - 2) / 255
|
||||
this.level.current = Math.min(1, avg * 1.9)
|
||||
this.raf = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
private stopMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
this.level.current = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Leichtgewichtige Stimmungs-Heuristik (v1) → treibt die Avatar-Mimik.
|
||||
// Bewusst simpel/regelbasiert (kein Modell): mappt deutschen Antworttext auf eine VRM-Expression.
|
||||
// Spätere Stufe: echte Hermes-Emotion/Audio-Tags. Siehe Plan.
|
||||
|
||||
export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed"
|
||||
|
||||
const RULES: [Emotion, RegExp][] = [
|
||||
["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha|:\)|😊|😄|🎉)/i],
|
||||
["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i],
|
||||
["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i],
|
||||
["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i],
|
||||
]
|
||||
|
||||
export function sentimentToEmotion(text: string): Emotion {
|
||||
for (const [emo, rx] of RULES) if (rx.test(text)) return emo
|
||||
return "neutral"
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Push-to-talk-Aufnahme über MediaRecorder. `start` beim Drücken (Taste/Button), `stop` beim
|
||||
// Loslassen → fertiges Audio-Blob (webm/opus) geht an `onAudio`. Bewusst minimal & generisch.
|
||||
export function usePushToTalk(onAudio: (blob: Blob) => void) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recRef = useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = useRef<Blob[]>([])
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recRef.current) return
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
console.error("Mikrofon-Zugriff verweigert:", e)
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus")
|
||||
? "audio/webm;codecs=opus"
|
||||
: "audio/webm"
|
||||
const rec = new MediaRecorder(stream, { mimeType: mime })
|
||||
chunksRef.current = []
|
||||
rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) }
|
||||
rec.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: mime })
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current = null
|
||||
recRef.current = null
|
||||
setRecording(false)
|
||||
if (blob.size > 1200) onAudio(blob) // Mini-Blobs (Versehen) ignorieren
|
||||
}
|
||||
rec.start()
|
||||
recRef.current = rec
|
||||
setRecording(true)
|
||||
}, [onAudio])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
recRef.current?.stop()
|
||||
}, [])
|
||||
|
||||
useEffect(() => () => {
|
||||
recRef.current?.stop()
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
}, [])
|
||||
|
||||
return { recording, start, stop }
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { usePushToTalk } from "./usePushToTalk"
|
||||
import { AudioQueue } from "./audio"
|
||||
import { sentimentToEmotion, type Emotion } from "./sentiment"
|
||||
|
||||
// Orchestriert die ganze Voll-Duplex-Schleife im Browser:
|
||||
// PTT-Audio → /api/voice/stt → User-Text
|
||||
// → /api/voice/chat (SSE vom Hermes-Agenten, server-seitiger Verlauf via Session-Id)
|
||||
// → Antwort satzweise schneiden → /api/voice/tts je Satz → AudioQueue (Abspielen + Lippensync)
|
||||
// → Stimmung aus dem Antworttext → Avatar-Mimik
|
||||
//
|
||||
// Avatar-Anbindung ohne Re-Render: `audioLevel` (Mundöffnung) und `emotion` sind Mutable-Refs,
|
||||
// die der 3D-Avatar pro Frame liest.
|
||||
|
||||
export type VoiceStatus = "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error"
|
||||
export interface ChatMsg { role: "user" | "assistant"; text: string }
|
||||
|
||||
const SYSTEM_PROMPT =
|
||||
"Du sprichst per Sprache mit dem Nutzer. Antworte natürlich, freundlich und KNAPP in ganzen, " +
|
||||
"gut vorlesbaren Sätzen. Kein Markdown, keine Codeblöcke, keine Aufzählungszeichen, keine Emojis — " +
|
||||
"reiner Fließtext, den man laut vorlesen kann."
|
||||
|
||||
function getSessionId(): string {
|
||||
let id = localStorage.getItem("mc_voice_session")
|
||||
if (!id) {
|
||||
id = "voice-" + Math.random().toString(36).slice(2) + Date.now().toString(36)
|
||||
localStorage.setItem("mc_voice_session", id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
function readSettings() {
|
||||
return {
|
||||
engine: localStorage.getItem("mc_voice_engine") || "piper",
|
||||
voice: localStorage.getItem("mc_voice_voice") || "",
|
||||
}
|
||||
}
|
||||
|
||||
// Zerlegt einen wachsenden Text-Stream in fertige Sätze. Gibt komplette Sätze zurück und behält
|
||||
// den unvollständigen Rest. So kann das erste TTS schon starten, bevor die Antwort fertig ist.
|
||||
function splitSentences(buffer: string): { sentences: string[]; rest: string } {
|
||||
const sentences: string[] = []
|
||||
const rx = /[^.!?…]+[.!?…]+(\s|$)/g
|
||||
let last = 0
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = rx.exec(buffer))) {
|
||||
sentences.push(m[0].trim())
|
||||
last = rx.lastIndex
|
||||
}
|
||||
return { sentences, rest: buffer.slice(last) }
|
||||
}
|
||||
|
||||
export function useVoiceAgent() {
|
||||
const [status, setStatus] = useState<VoiceStatus>("idle")
|
||||
const [messages, setMessages] = useState<ChatMsg[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const audioLevel = useRef({ current: 0 }) // wird gleich auf die Queue-Pegel gezeigt
|
||||
const emotion = useRef<Emotion>("neutral")
|
||||
const queueRef = useRef<AudioQueue | null>(null)
|
||||
const sessionId = useRef<string>(getSessionId())
|
||||
|
||||
// AudioQueue erst bei Bedarf (nach User-Geste) erzeugen — Autoplay-Policy.
|
||||
const ensureQueue = useCallback(() => {
|
||||
if (!queueRef.current) {
|
||||
const q = new AudioQueue()
|
||||
q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s))
|
||||
queueRef.current = q
|
||||
audioLevel.current = q.level // Avatar liest ab jetzt echte Pegel
|
||||
}
|
||||
return queueRef.current
|
||||
}, [])
|
||||
|
||||
const handleAudio = useCallback(async (blob: Blob) => {
|
||||
setError(null)
|
||||
const queue = ensureQueue()
|
||||
queue.clear() // evtl. laufende Antwort abbrechen (Barge-in)
|
||||
|
||||
// 1) STT
|
||||
setStatus("transcribing")
|
||||
let userText = ""
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", blob, "rec.webm")
|
||||
const r = await fetch("/api/voice/stt", { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`STT ${r.status}`)
|
||||
userText = (await r.json()).text?.trim() || ""
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return
|
||||
}
|
||||
if (!userText) { setStatus("idle"); return }
|
||||
setMessages((m) => [...m, { role: "user", text: userText }])
|
||||
|
||||
// 2) Chat (SSE) → 3) satzweises TTS
|
||||
setStatus("thinking")
|
||||
const { engine, voice } = readSettings()
|
||||
let assistant = ""
|
||||
let pending = ""
|
||||
setMessages((m) => [...m, { role: "assistant", text: "" }])
|
||||
|
||||
const speak = (sentence: string) => {
|
||||
if (!sentence.trim()) return
|
||||
fetch("/api/voice/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: sentence, engine, voice }),
|
||||
})
|
||||
.then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(`TTS ${r.status}`))))
|
||||
.then((buf) => queue.enqueue(buf))
|
||||
.catch((e) => console.error("TTS-Fehler:", e))
|
||||
}
|
||||
|
||||
try {
|
||||
const r = await fetch("/api/voice/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
text: userText,
|
||||
session_id: sessionId.current,
|
||||
system: SYSTEM_PROMPT,
|
||||
}),
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`)
|
||||
|
||||
const reader = r.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let sse = ""
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
sse += dec.decode(value, { stream: true })
|
||||
const events = sse.split("\n\n")
|
||||
sse = events.pop() || ""
|
||||
for (const ev of events) {
|
||||
const line = ev.split("\n").find((l) => l.startsWith("data:"))
|
||||
if (!line) continue
|
||||
const data = line.slice(5).trim()
|
||||
if (data === "[DONE]") continue
|
||||
let json: any
|
||||
try { json = JSON.parse(data) } catch { continue }
|
||||
if (json.error) throw new Error(json.error)
|
||||
const delta = json.choices?.[0]?.delta?.content || ""
|
||||
if (!delta) continue
|
||||
assistant += delta
|
||||
pending += delta
|
||||
emotion.current = sentimentToEmotion(assistant)
|
||||
setMessages((m) => {
|
||||
const copy = m.slice()
|
||||
copy[copy.length - 1] = { role: "assistant", text: assistant }
|
||||
return copy
|
||||
})
|
||||
const { sentences, rest } = splitSentences(pending)
|
||||
pending = rest
|
||||
sentences.forEach(speak)
|
||||
}
|
||||
}
|
||||
if (pending.trim()) speak(pending) // Rest (letzter Satz ohne Satzzeichen)
|
||||
if (!assistant.trim()) setStatus("idle")
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`)
|
||||
}
|
||||
}, [ensureQueue])
|
||||
|
||||
const { recording, start, stop } = usePushToTalk(handleAudio)
|
||||
|
||||
const pressStart = useCallback(() => {
|
||||
ensureQueue()
|
||||
setStatus("listening")
|
||||
void start()
|
||||
}, [ensureQueue, start])
|
||||
|
||||
const pressEnd = useCallback(() => { stop() }, [stop])
|
||||
|
||||
const reset = useCallback(() => {
|
||||
queueRef.current?.clear()
|
||||
setMessages([])
|
||||
setError(null)
|
||||
setStatus("idle")
|
||||
localStorage.removeItem("mc_voice_session")
|
||||
sessionId.current = getSessionId()
|
||||
}, [])
|
||||
|
||||
// Leerlauf-Status zurücksetzen, wenn nichts mehr spricht/aufnimmt.
|
||||
useEffect(() => {
|
||||
if (!recording && (status === "listening")) setStatus("transcribing")
|
||||
}, [recording, status])
|
||||
|
||||
return {
|
||||
status, messages, error, recording,
|
||||
audioLevel, emotion,
|
||||
pressStart, pressEnd, reset,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Winziger IndexedDB-Wrapper, um ein hochgeladenes .vrm (ArrayBuffer) über Reloads hinweg zu
|
||||
// behalten. localStorage scheidet aus (VRMs sind oft 5–40 MB). Ein einziger Record genügt.
|
||||
|
||||
const DB = "mc2-voice"
|
||||
const STORE = "avatar"
|
||||
const KEY = "uploaded-vrm"
|
||||
|
||||
function open(): Promise<IDBDatabase> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = indexedDB.open(DB, 1)
|
||||
req.onupgradeneeded = () => req.result.createObjectStore(STORE)
|
||||
req.onsuccess = () => resolve(req.result)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function saveUploadedVrm(buf: ArrayBuffer): Promise<void> {
|
||||
const db = await open()
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readwrite")
|
||||
tx.objectStore(STORE).put(buf, KEY)
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error)
|
||||
})
|
||||
}
|
||||
|
||||
export async function loadUploadedVrm(): Promise<ArrayBuffer | null> {
|
||||
const db = await open()
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, "readonly")
|
||||
const req = tx.objectStore(STORE).get(KEY)
|
||||
req.onsuccess = () => resolve((req.result as ArrayBuffer) ?? null)
|
||||
req.onerror = () => reject(req.error)
|
||||
})
|
||||
}
|
||||
+3
-1
@@ -6,10 +6,11 @@ import {
|
||||
Bot,
|
||||
TerminalSquare,
|
||||
HelpCircle,
|
||||
Mic,
|
||||
type LucideIcon,
|
||||
} from "lucide-react"
|
||||
|
||||
export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "guide"
|
||||
export type ViewId = "dashboard" | "models" | "memory" | "connect" | "agent" | "terminal" | "voice" | "guide"
|
||||
|
||||
export interface NavItem {
|
||||
id: ViewId
|
||||
@@ -26,6 +27,7 @@ export const NAV: NavItem[] = [
|
||||
{ id: "connect", label: "Verbinden", hint: "IDE-/Agent-Configs erzeugen", icon: Plug },
|
||||
{ id: "agent", label: "Hermes", hint: "Agent-Status & Verdrahtung", icon: Bot },
|
||||
{ id: "terminal", label: "Terminal", hint: "Interaktives Hermes-Agent-Terminal", icon: TerminalSquare },
|
||||
{ id: "voice", label: "Sprechen", hint: "Mit Hermes per Sprache reden (3D-Avatar)", icon: Mic },
|
||||
{ id: "guide", label: "Anleitung", hint: "Einrichten & Vibe-Coding", icon: HelpCircle },
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { Mic, RotateCcw, Loader2, Volume2 } from "lucide-react"
|
||||
import { Avatar3D } from "@/components/voice/Avatar3D"
|
||||
import { AvatarPicker, DEFAULT_AVATAR } from "@/components/voice/AvatarPicker"
|
||||
import { useVoiceAgent } from "@/lib/voice/useVoiceAgent"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
idle: "Bereit — halte zum Sprechen",
|
||||
listening: "Höre zu …",
|
||||
transcribing: "Verstehe …",
|
||||
thinking: "Hermes denkt …",
|
||||
speaking: "Hermes spricht …",
|
||||
error: "Fehler",
|
||||
}
|
||||
|
||||
export function VoiceView() {
|
||||
const [avatarUrl, setAvatarUrl] = useState(
|
||||
() => localStorage.getItem("mc_voice_avatar_url") || DEFAULT_AVATAR,
|
||||
)
|
||||
const { status, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset } =
|
||||
useVoiceAgent()
|
||||
|
||||
const holding = useRef(false)
|
||||
const onAvatarChange = useCallback((url: string) => setAvatarUrl(url), [])
|
||||
|
||||
// Push-to-talk per Leertaste (solange der Sprechen-Tab fokussiert ist und kein Eingabefeld aktiv).
|
||||
useEffect(() => {
|
||||
const isField = (el: EventTarget | null) =>
|
||||
el instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName)
|
||||
const down = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || e.repeat || holding.current || isField(e.target)) return
|
||||
e.preventDefault(); holding.current = true; pressStart()
|
||||
}
|
||||
const up = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || !holding.current) return
|
||||
e.preventDefault(); holding.current = false; pressEnd()
|
||||
}
|
||||
window.addEventListener("keydown", down)
|
||||
window.addEventListener("keyup", up)
|
||||
return () => { window.removeEventListener("keydown", down); window.removeEventListener("keyup", up) }
|
||||
}, [pressStart, pressEnd])
|
||||
|
||||
const speaking = status === "speaking"
|
||||
const busy = status === "transcribing" || status === "thinking"
|
||||
|
||||
return (
|
||||
<div className="flex h-full gap-5">
|
||||
{/* Avatar-Bühne */}
|
||||
<div className="relative flex min-w-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/30 overflow-hidden">
|
||||
<div className="flex-1 min-h-0">
|
||||
<Avatar3D url={avatarUrl} audioLevel={audioLevel} emotion={emotion} />
|
||||
</div>
|
||||
|
||||
{/* Status + Push-to-talk */}
|
||||
<div className="shrink-0 flex flex-col items-center gap-3 border-t border-border/40 bg-background/30 py-5 backdrop-blur-sm">
|
||||
<div className={cn(
|
||||
"flex items-center gap-2 text-sm",
|
||||
status === "error" ? "text-red-400" : speaking ? "text-primary" : "text-muted-foreground",
|
||||
)}>
|
||||
{busy && <Loader2 className="h-4 w-4 animate-spin" />}
|
||||
{speaking && <Volume2 className="h-4 w-4 animate-pulse" />}
|
||||
<span>{error || STATUS_LABEL[status]}</span>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onPointerDown={(e) => { e.preventDefault(); holding.current = true; pressStart() }}
|
||||
onPointerUp={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
onPointerLeave={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
className={cn(
|
||||
"flex h-20 w-20 items-center justify-center rounded-full border-2 transition-all select-none touch-none",
|
||||
recording
|
||||
? "border-red-500 bg-red-500/20 scale-110 shadow-lg shadow-red-500/30"
|
||||
: "border-primary/60 bg-primary/15 hover:bg-primary/25 hover:scale-105",
|
||||
)}
|
||||
title="Gedrückt halten zum Sprechen (oder Leertaste halten)"
|
||||
>
|
||||
<Mic className={cn("h-8 w-8", recording ? "text-red-400" : "text-primary")} />
|
||||
</button>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Halten zum Sprechen · <kbd className="rounded bg-muted px-1 py-0.5 font-mono">Leertaste</kbd> geht auch
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Seitenspalte: Einstellungen + Transcript */}
|
||||
<div className="flex w-80 shrink-0 flex-col gap-4">
|
||||
<div className="rounded-xl border border-border/40 bg-card/40 p-4 overflow-y-auto scrollbar-thin max-h-[55%]">
|
||||
<AvatarPicker avatarUrl={avatarUrl} onAvatarChange={onAvatarChange} />
|
||||
</div>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/40">
|
||||
<div className="flex items-center justify-between border-b border-border/40 px-4 py-2.5">
|
||||
<span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Gespräch</span>
|
||||
<button onClick={reset} className="text-muted-foreground hover:text-foreground" title="Neues Gespräch">
|
||||
<RotateCcw className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex-1 space-y-2.5 overflow-y-auto p-4 scrollbar-thin">
|
||||
{messages.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Halte den Knopf (oder die Leertaste) und sprich. Hermes hört zu, denkt mit seinem
|
||||
vollen Gedächtnis und seinen Werkzeugen — und antwortet hörbar.
|
||||
</p>
|
||||
)}
|
||||
{messages.map((m, i) => (
|
||||
<div key={i} className={cn("text-sm", m.role === "user" ? "text-foreground" : "text-primary/90")}>
|
||||
<span className="mr-1.5 text-[10px] font-semibold uppercase text-muted-foreground">
|
||||
{m.role === "user" ? "Du" : "Hermes"}
|
||||
</span>
|
||||
{m.text || <span className="text-muted-foreground">…</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user