Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,184 @@
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { Upload, Link2, ExternalLink, Check, Sparkles } from "lucide-react"
|
||||
import { saveUploadedVrm, loadUploadedVrm } from "@/lib/voice/vrmStore"
|
||||
|
||||
// Avatar selbst aussuchen: kuratierte Galerie (öffentliche, CORS-freie VRMs) + eigenes .vrm
|
||||
// hochladen + per URL laden + VRoid-Hub-Link. Dazu die Stimm-Auswahl (Engine + Stimme).
|
||||
// Auswahl bleibt in localStorage / IndexedDB erhalten.
|
||||
|
||||
interface GalleryItem { id: string; label: string; url: string; note?: string }
|
||||
|
||||
// Verifiziert: 200 + Access-Control-Allow-Origin:* (im Browser ladbar).
|
||||
const GALLERY: GalleryItem[] = [
|
||||
{ id: "sample-a", label: "VRoid Sample A", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_A.vrm", note: "Anime, weiblich" },
|
||||
{ id: "sample-b", label: "VRoid Sample B", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_B.vrm", note: "Anime, männlich" },
|
||||
{ id: "pixiv", label: "Pixiv Demo", url: "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm", note: "VRM1-Testmodell" },
|
||||
]
|
||||
|
||||
export const DEFAULT_AVATAR = GALLERY[0].url
|
||||
|
||||
interface Voice { engine: string; id: string; label: string; clonable?: boolean }
|
||||
|
||||
export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
avatarUrl: string; onAvatarChange: (url: string) => void
|
||||
}) {
|
||||
const [urlInput, setUrlInput] = useState("")
|
||||
const [voices, setVoices] = useState<Voice[]>([])
|
||||
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
|
||||
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
|
||||
const fileRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
// Hochgeladenes VRM nach Reload wiederherstellen.
|
||||
useEffect(() => {
|
||||
if (localStorage.getItem("mc_voice_avatar_uploaded") === "1") {
|
||||
loadUploadedVrm().then((buf) => {
|
||||
if (buf) onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
})
|
||||
}
|
||||
}, [onAvatarChange])
|
||||
|
||||
// Stimmen vom Sidecar holen.
|
||||
useEffect(() => {
|
||||
fetch("/api/voice/voices")
|
||||
.then((r) => (r.ok ? r.json() : Promise.reject()))
|
||||
.then((d) => setVoices(d.voices || []))
|
||||
.catch(() => setVoices([]))
|
||||
}, [])
|
||||
|
||||
const pickGallery = (url: string) => {
|
||||
localStorage.setItem("mc_voice_avatar_url", url)
|
||||
localStorage.removeItem("mc_voice_avatar_uploaded")
|
||||
onAvatarChange(url)
|
||||
}
|
||||
|
||||
const onUpload = async (file: File) => {
|
||||
const buf = await file.arrayBuffer()
|
||||
await saveUploadedVrm(buf)
|
||||
localStorage.setItem("mc_voice_avatar_uploaded", "1")
|
||||
localStorage.removeItem("mc_voice_avatar_url")
|
||||
onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
}
|
||||
|
||||
const loadUrl = () => {
|
||||
const u = urlInput.trim()
|
||||
if (u) pickGallery(u)
|
||||
}
|
||||
|
||||
const saveVoice = (eng: string, v: string) => {
|
||||
setEngine(eng); setVoice(v)
|
||||
localStorage.setItem("mc_voice_engine", eng)
|
||||
localStorage.setItem("mc_voice_voice", v)
|
||||
}
|
||||
|
||||
const enginesAvail = Array.from(new Set(voices.map((v) => v.engine)))
|
||||
const voicesForEngine = voices.filter((v) => v.engine === engine)
|
||||
|
||||
return (
|
||||
<div className="space-y-5 text-sm">
|
||||
{/* Galerie */}
|
||||
<div>
|
||||
<div className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">Avatar</div>
|
||||
<div className="space-y-1.5">
|
||||
{GALLERY.map((g) => {
|
||||
const active = avatarUrl === g.url
|
||||
return (
|
||||
<button
|
||||
key={g.id}
|
||||
onClick={() => pickGallery(g.url)}
|
||||
className={`flex w-full items-center justify-between rounded-md border px-3 py-2 text-left transition-colors ${
|
||||
active ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
<span>
|
||||
<span className="font-medium">{g.label}</span>
|
||||
{g.note && <span className="ml-2 text-[11px] text-muted-foreground">{g.note}</span>}
|
||||
</span>
|
||||
{active && <Check className="h-4 w-4" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Eigenes Modell */}
|
||||
<div className="space-y-2">
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept=".vrm,model/gltf-binary"
|
||||
className="hidden"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) void onUpload(f) }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => fileRef.current?.click()}
|
||||
className="flex w-full items-center gap-2 rounded-md border border-border/40 bg-background/40 px-3 py-2 hover:bg-accent transition-colors"
|
||||
>
|
||||
<Upload className="h-4 w-4" /> Eigenes .vrm hochladen
|
||||
</button>
|
||||
<div className="flex gap-1.5">
|
||||
<input
|
||||
value={urlInput}
|
||||
onChange={(e) => setUrlInput(e.target.value)}
|
||||
onKeyDown={(e) => e.key === "Enter" && loadUrl()}
|
||||
placeholder="…oder VRM-URL einfügen"
|
||||
className="min-w-0 flex-1 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
/>
|
||||
<button onClick={loadUrl} className="rounded-md border border-border/40 bg-background/40 px-2.5 hover:bg-accent" title="Laden">
|
||||
<Link2 className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
<a
|
||||
href="https://hub.vroid.com/en/characters"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
className="flex items-center gap-1.5 text-xs text-primary/80 hover:text-primary"
|
||||
>
|
||||
<ExternalLink className="h-3.5 w-3.5" /> Mehr Avatare auf VRoid Hub (kostenlos) → herunterladen & hochladen
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{/* Stimme */}
|
||||
<div>
|
||||
<div className="mb-2 flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||
<Sparkles className="h-3.5 w-3.5" /> Stimme
|
||||
</div>
|
||||
{voices.length === 0 ? (
|
||||
<div className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-300">
|
||||
Voice-Dienst nicht erreichbar — Stimmen werden geladen, sobald der Sidecar läuft.
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="flex gap-1.5">
|
||||
{enginesAvail.map((eng) => (
|
||||
<button
|
||||
key={eng}
|
||||
onClick={() => saveVoice(eng, "")}
|
||||
className={`flex-1 rounded-md border px-2.5 py-1.5 text-xs capitalize transition-colors ${
|
||||
engine === eng ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
{eng === "piper" ? "Piper (schnell)" : eng === "chatterbox" ? "Chatterbox (premium)" : eng}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<select
|
||||
value={voice}
|
||||
onChange={(e) => saveVoice(engine, e.target.value)}
|
||||
className="w-full rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
>
|
||||
<option value="">Standardstimme</option>
|
||||
{voicesForEngine.map((v) => (
|
||||
<option key={v.id} value={v.id}>{v.label}{v.clonable ? " · klonbar" : ""}</option>
|
||||
))}
|
||||
</select>
|
||||
{engine === "chatterbox" && (
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Natürlichste Stimme + Voice-Cloning.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user