Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,119 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { Mic, RotateCcw, Loader2, Volume2 } from "lucide-react"
|
||||
import { Avatar3D } from "@/components/voice/Avatar3D"
|
||||
import { AvatarPicker, DEFAULT_AVATAR } from "@/components/voice/AvatarPicker"
|
||||
import { useVoiceAgent } from "@/lib/voice/useVoiceAgent"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
idle: "Bereit — halte zum Sprechen",
|
||||
listening: "Höre zu …",
|
||||
transcribing: "Verstehe …",
|
||||
thinking: "Hermes denkt …",
|
||||
speaking: "Hermes spricht …",
|
||||
error: "Fehler",
|
||||
}
|
||||
|
||||
export function VoiceView() {
|
||||
const [avatarUrl, setAvatarUrl] = useState(
|
||||
() => localStorage.getItem("mc_voice_avatar_url") || DEFAULT_AVATAR,
|
||||
)
|
||||
const { status, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset } =
|
||||
useVoiceAgent()
|
||||
|
||||
const holding = useRef(false)
|
||||
const onAvatarChange = useCallback((url: string) => setAvatarUrl(url), [])
|
||||
|
||||
// Push-to-talk per Leertaste (solange der Sprechen-Tab fokussiert ist und kein Eingabefeld aktiv).
|
||||
useEffect(() => {
|
||||
const isField = (el: EventTarget | null) =>
|
||||
el instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName)
|
||||
const down = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || e.repeat || holding.current || isField(e.target)) return
|
||||
e.preventDefault(); holding.current = true; pressStart()
|
||||
}
|
||||
const up = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || !holding.current) return
|
||||
e.preventDefault(); holding.current = false; pressEnd()
|
||||
}
|
||||
window.addEventListener("keydown", down)
|
||||
window.addEventListener("keyup", up)
|
||||
return () => { window.removeEventListener("keydown", down); window.removeEventListener("keyup", up) }
|
||||
}, [pressStart, pressEnd])
|
||||
|
||||
const speaking = status === "speaking"
|
||||
const busy = status === "transcribing" || status === "thinking"
|
||||
|
||||
return (
|
||||
<div className="flex h-full gap-5">
|
||||
{/* Avatar-Bühne */}
|
||||
<div className="relative flex min-w-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/30 overflow-hidden">
|
||||
<div className="flex-1 min-h-0">
|
||||
<Avatar3D url={avatarUrl} audioLevel={audioLevel} emotion={emotion} />
|
||||
</div>
|
||||
|
||||
{/* Status + Push-to-talk */}
|
||||
<div className="shrink-0 flex flex-col items-center gap-3 border-t border-border/40 bg-background/30 py-5 backdrop-blur-sm">
|
||||
<div className={cn(
|
||||
"flex items-center gap-2 text-sm",
|
||||
status === "error" ? "text-red-400" : speaking ? "text-primary" : "text-muted-foreground",
|
||||
)}>
|
||||
{busy && <Loader2 className="h-4 w-4 animate-spin" />}
|
||||
{speaking && <Volume2 className="h-4 w-4 animate-pulse" />}
|
||||
<span>{error || STATUS_LABEL[status]}</span>
|
||||
</div>
|
||||
|
||||
<button
|
||||
onPointerDown={(e) => { e.preventDefault(); holding.current = true; pressStart() }}
|
||||
onPointerUp={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
onPointerLeave={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
className={cn(
|
||||
"flex h-20 w-20 items-center justify-center rounded-full border-2 transition-all select-none touch-none",
|
||||
recording
|
||||
? "border-red-500 bg-red-500/20 scale-110 shadow-lg shadow-red-500/30"
|
||||
: "border-primary/60 bg-primary/15 hover:bg-primary/25 hover:scale-105",
|
||||
)}
|
||||
title="Gedrückt halten zum Sprechen (oder Leertaste halten)"
|
||||
>
|
||||
<Mic className={cn("h-8 w-8", recording ? "text-red-400" : "text-primary")} />
|
||||
</button>
|
||||
<div className="text-[11px] text-muted-foreground">
|
||||
Halten zum Sprechen · <kbd className="rounded bg-muted px-1 py-0.5 font-mono">Leertaste</kbd> geht auch
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Seitenspalte: Einstellungen + Transcript */}
|
||||
<div className="flex w-80 shrink-0 flex-col gap-4">
|
||||
<div className="rounded-xl border border-border/40 bg-card/40 p-4 overflow-y-auto scrollbar-thin max-h-[55%]">
|
||||
<AvatarPicker avatarUrl={avatarUrl} onAvatarChange={onAvatarChange} />
|
||||
</div>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col rounded-xl border border-border/40 bg-card/40">
|
||||
<div className="flex items-center justify-between border-b border-border/40 px-4 py-2.5">
|
||||
<span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Gespräch</span>
|
||||
<button onClick={reset} className="text-muted-foreground hover:text-foreground" title="Neues Gespräch">
|
||||
<RotateCcw className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex-1 space-y-2.5 overflow-y-auto p-4 scrollbar-thin">
|
||||
{messages.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Halte den Knopf (oder die Leertaste) und sprich. Hermes hört zu, denkt mit seinem
|
||||
vollen Gedächtnis und seinen Werkzeugen — und antwortet hörbar.
|
||||
</p>
|
||||
)}
|
||||
{messages.map((m, i) => (
|
||||
<div key={i} className={cn("text-sm", m.role === "user" ? "text-foreground" : "text-primary/90")}>
|
||||
<span className="mr-1.5 text-[10px] font-semibold uppercase text-muted-foreground">
|
||||
{m.role === "user" ? "Du" : "Hermes"}
|
||||
</span>
|
||||
{m.text || <span className="text-muted-foreground">…</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user