Feat: Sprechen-Tab — mit Hermes per Sprache reden (Browser-Voice + 3D-Avatar)
Voll-Duplex Sprach-Interaktion vom lokalen PC mit dem vollen Hermes-Agenten (api_server :8642, OpenAI-kompatibel → gleiche Tools + geteiltes Mem0 wie CLI/Telegram). - Voice-Sidecar (voice_service/, eigenes Py3.12-venv ~/.voice, :8650): STT faster-whisper (medium, de) + gestuftes TTS — Piper (schnell, Default) + Chatterbox (premium, Voice-Cloning, lazy-load, CPU-Start). Analog mem0_service. - Backend: routers/voice.py (Proxy /api/voice/stt|tts|voices + /chat-SSE an Hermes mit Bearer API_SERVER_KEY + X-Hermes-Session-Id für server-seitigen Verlauf). config.py: VOICE_SERVICE_URL + HERMES_API_KEY (Fallback aus ~/.hermes/.env). System-Dienstliste + Wartung (Restart/Logs) um voice-service ergänzt. - Frontend: Sprechen-Tab mit 3D-Avatar (VRM via three-vrm) — Lippensync (Web-Audio-Pegel), Blinzeln, Sentiment-Mimik, Ruhepose. Avatar-Picker (CORS-freie Galerie + .vrm-Upload + URL + VRoid-Hub-Link) + Stimm-Auswahl. Push-to-talk (Knopf/Leertaste). Deps: three, r3f, drei. - Deploy: deploy/voice-service.service + deploy.sh (idempotenter Sidecar-Install, enable, restart). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,139 @@
|
||||
import { Canvas, useFrame } from "@react-three/fiber"
|
||||
import { OrbitControls } from "@react-three/drei"
|
||||
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
||||
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
||||
import type { Emotion } from "@/lib/voice/sentiment"
|
||||
|
||||
// 3D-Avatar (VRM) mit Lippensync (Mund folgt dem TTS-Audiopegel), automatischem Blinzeln und
|
||||
// stimmungsabhängiger Mimik. Liest die Live-Werte aus Mutable-Refs (kein Re-Render pro Frame).
|
||||
|
||||
type LevelRef = MutableRefObject<{ current: number }> // audioLevel.current.current = Pegel 0..1
|
||||
type EmotionRef = MutableRefObject<Emotion>
|
||||
|
||||
// VRMs laden in T-Pose (Bindepose, Arme waagerecht). Wir senken Ober-/Unterarme auf den
|
||||
// normalisierten Humanoid-Knoten zu einer ruhigen A-Pose ab — sieht sofort natürlich aus.
|
||||
// (Echte Idle-Animation wäre die spätere Stufe.)
|
||||
function applyRestPose(vrm: VRM) {
|
||||
const set = (name: any, x: number, y: number, z: number) => {
|
||||
const b = vrm.humanoid?.getNormalizedBoneNode(name)
|
||||
if (b) b.rotation.set(x, y, z)
|
||||
}
|
||||
set("leftUpperArm", 0, 0, 1.2) // Arm runter an die Seite
|
||||
set("rightUpperArm", 0, 0, -1.2)
|
||||
set("leftLowerArm", 0, -0.2, 0) // leichte Beugung
|
||||
set("rightLowerArm", 0, 0.2, 0)
|
||||
vrm.humanoid?.update()
|
||||
}
|
||||
|
||||
const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
|
||||
const EMO_TO_EXPR: Record<Emotion, string | null> = {
|
||||
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
|
||||
}
|
||||
|
||||
function VrmModel({ url, audioLevel, emotion, onError }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef; onError: (m: string) => void
|
||||
}) {
|
||||
const [vrm, setVrm] = useState<VRM | null>(null)
|
||||
const smooth = useRef<Record<string, number>>({})
|
||||
const blink = useRef({ t: 0, next: 3, active: 0 })
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false
|
||||
let loaded: VRM | null = null
|
||||
const loader = new GLTFLoader()
|
||||
loader.register((parser) => new VRMLoaderPlugin(parser))
|
||||
loader.load(
|
||||
url,
|
||||
(gltf) => {
|
||||
if (disposed) return
|
||||
const v = gltf.userData.vrm as VRM | undefined
|
||||
if (!v) { onError("Datei enthält kein gültiges VRM-Modell."); return }
|
||||
VRMUtils.removeUnnecessaryVertices(gltf.scene)
|
||||
if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
|
||||
v.scene.rotation.y = Math.PI // dem Betrachter zuwenden
|
||||
applyRestPose(v) // T-Pose → entspannte A-Pose (Arme unten)
|
||||
loaded = v
|
||||
setVrm(v)
|
||||
},
|
||||
undefined,
|
||||
(err) => { console.error("VRM-Load-Fehler:", err); onError("Avatar konnte nicht geladen werden (CORS/URL?).") },
|
||||
)
|
||||
return () => {
|
||||
disposed = true
|
||||
if (loaded) VRMUtils.deepDispose(loaded.scene)
|
||||
setVrm(null)
|
||||
}
|
||||
}, [url, onError])
|
||||
|
||||
useFrame((_, delta) => {
|
||||
if (!vrm) return
|
||||
const em = vrm.expressionManager
|
||||
if (em) {
|
||||
// Lippensync: 'aa' folgt geglättet dem Audiopegel.
|
||||
const target = audioLevel.current?.current ?? 0
|
||||
const aa = (smooth.current.aa ?? 0) * 0.4 + target * 0.6
|
||||
smooth.current.aa = aa
|
||||
em.setValue("aa", aa)
|
||||
|
||||
// Mimik: weich zur Ziel-Expression lerpen.
|
||||
const want = EMO_TO_EXPR[emotion.current]
|
||||
for (const name of EXPRESSIONS) {
|
||||
const tv = want === name ? 0.75 : 0
|
||||
const cv = smooth.current[name] ?? 0
|
||||
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
||||
smooth.current[name] = nv
|
||||
em.setValue(name, nv)
|
||||
}
|
||||
|
||||
// Blinzeln: kurzer Dreieckspuls alle 3–7 s.
|
||||
const b = blink.current
|
||||
b.t += delta
|
||||
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
||||
let blinkVal = 0
|
||||
if (b.active > 0) {
|
||||
b.active -= delta
|
||||
const p = 1 - b.active / 0.16 // 0..1 Fortschritt
|
||||
blinkVal = 1 - Math.abs(p - 0.5) * 2 // 0 → 1 → 0
|
||||
}
|
||||
em.setValue("blink", Math.max(0, blinkVal))
|
||||
}
|
||||
vrm.update(delta)
|
||||
})
|
||||
|
||||
return vrm ? <primitive object={vrm.scene} /> : null
|
||||
}
|
||||
|
||||
export function Avatar3D({ url, audioLevel, emotion }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef
|
||||
}) {
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
return (
|
||||
<div className="relative h-full w-full">
|
||||
<Canvas
|
||||
camera={{ position: [0, 1.35, 1.25], fov: 30 }}
|
||||
gl={{ alpha: true, antialias: true }}
|
||||
style={{ background: "transparent" }}
|
||||
>
|
||||
<ambientLight intensity={0.85} />
|
||||
<directionalLight position={[1, 2, 2]} intensity={1.1} />
|
||||
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
|
||||
{/* key=url → bei Avatarwechsel sauber neu mounten */}
|
||||
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} onError={setErr} />
|
||||
<OrbitControls
|
||||
target={[0, 1.3, 0]}
|
||||
enablePan={false}
|
||||
minDistance={0.7}
|
||||
maxDistance={3}
|
||||
minPolarAngle={Math.PI / 3}
|
||||
maxPolarAngle={Math.PI / 1.8}
|
||||
/>
|
||||
</Canvas>
|
||||
{err && (
|
||||
<div className="absolute inset-x-0 bottom-3 mx-auto w-fit rounded-md bg-red-500/15 border border-red-500/30 px-3 py-1.5 text-xs text-red-300">
|
||||
{err}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { Upload, Link2, ExternalLink, Check, Sparkles } from "lucide-react"
|
||||
import { saveUploadedVrm, loadUploadedVrm } from "@/lib/voice/vrmStore"
|
||||
|
||||
// Avatar selbst aussuchen: kuratierte Galerie (öffentliche, CORS-freie VRMs) + eigenes .vrm
|
||||
// hochladen + per URL laden + VRoid-Hub-Link. Dazu die Stimm-Auswahl (Engine + Stimme).
|
||||
// Auswahl bleibt in localStorage / IndexedDB erhalten.
|
||||
|
||||
interface GalleryItem { id: string; label: string; url: string; note?: string }
|
||||
|
||||
// Verifiziert: 200 + Access-Control-Allow-Origin:* (im Browser ladbar).
|
||||
const GALLERY: GalleryItem[] = [
|
||||
{ id: "sample-a", label: "VRoid Sample A", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_A.vrm", note: "Anime, weiblich" },
|
||||
{ id: "sample-b", label: "VRoid Sample B", url: "https://raw.githubusercontent.com/madjin/vrm-samples/master/vroid/stable/AvatarSample_B.vrm", note: "Anime, männlich" },
|
||||
{ id: "pixiv", label: "Pixiv Demo", url: "https://raw.githubusercontent.com/pixiv/three-vrm/dev/packages/three-vrm/examples/models/VRM1_Constraint_Twist_Sample.vrm", note: "VRM1-Testmodell" },
|
||||
]
|
||||
|
||||
export const DEFAULT_AVATAR = GALLERY[0].url
|
||||
|
||||
interface Voice { engine: string; id: string; label: string; clonable?: boolean }
|
||||
|
||||
export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
avatarUrl: string; onAvatarChange: (url: string) => void
|
||||
}) {
|
||||
const [urlInput, setUrlInput] = useState("")
|
||||
const [voices, setVoices] = useState<Voice[]>([])
|
||||
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
|
||||
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
|
||||
const fileRef = useRef<HTMLInputElement>(null)
|
||||
|
||||
// Hochgeladenes VRM nach Reload wiederherstellen.
|
||||
useEffect(() => {
|
||||
if (localStorage.getItem("mc_voice_avatar_uploaded") === "1") {
|
||||
loadUploadedVrm().then((buf) => {
|
||||
if (buf) onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
})
|
||||
}
|
||||
}, [onAvatarChange])
|
||||
|
||||
// Stimmen vom Sidecar holen.
|
||||
useEffect(() => {
|
||||
fetch("/api/voice/voices")
|
||||
.then((r) => (r.ok ? r.json() : Promise.reject()))
|
||||
.then((d) => setVoices(d.voices || []))
|
||||
.catch(() => setVoices([]))
|
||||
}, [])
|
||||
|
||||
const pickGallery = (url: string) => {
|
||||
localStorage.setItem("mc_voice_avatar_url", url)
|
||||
localStorage.removeItem("mc_voice_avatar_uploaded")
|
||||
onAvatarChange(url)
|
||||
}
|
||||
|
||||
const onUpload = async (file: File) => {
|
||||
const buf = await file.arrayBuffer()
|
||||
await saveUploadedVrm(buf)
|
||||
localStorage.setItem("mc_voice_avatar_uploaded", "1")
|
||||
localStorage.removeItem("mc_voice_avatar_url")
|
||||
onAvatarChange(URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })))
|
||||
}
|
||||
|
||||
const loadUrl = () => {
|
||||
const u = urlInput.trim()
|
||||
if (u) pickGallery(u)
|
||||
}
|
||||
|
||||
const saveVoice = (eng: string, v: string) => {
|
||||
setEngine(eng); setVoice(v)
|
||||
localStorage.setItem("mc_voice_engine", eng)
|
||||
localStorage.setItem("mc_voice_voice", v)
|
||||
}
|
||||
|
||||
const enginesAvail = Array.from(new Set(voices.map((v) => v.engine)))
|
||||
const voicesForEngine = voices.filter((v) => v.engine === engine)
|
||||
|
||||
return (
|
||||
<div className="space-y-5 text-sm">
|
||||
{/* Galerie */}
|
||||
<div>
|
||||
<div className="mb-2 text-xs font-semibold uppercase tracking-wide text-muted-foreground">Avatar</div>
|
||||
<div className="space-y-1.5">
|
||||
{GALLERY.map((g) => {
|
||||
const active = avatarUrl === g.url
|
||||
return (
|
||||
<button
|
||||
key={g.id}
|
||||
onClick={() => pickGallery(g.url)}
|
||||
className={`flex w-full items-center justify-between rounded-md border px-3 py-2 text-left transition-colors ${
|
||||
active ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
<span>
|
||||
<span className="font-medium">{g.label}</span>
|
||||
{g.note && <span className="ml-2 text-[11px] text-muted-foreground">{g.note}</span>}
|
||||
</span>
|
||||
{active && <Check className="h-4 w-4" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Eigenes Modell */}
|
||||
<div className="space-y-2">
|
||||
<input
|
||||
ref={fileRef}
|
||||
type="file"
|
||||
accept=".vrm,model/gltf-binary"
|
||||
className="hidden"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) void onUpload(f) }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => fileRef.current?.click()}
|
||||
className="flex w-full items-center gap-2 rounded-md border border-border/40 bg-background/40 px-3 py-2 hover:bg-accent transition-colors"
|
||||
>
|
||||
<Upload className="h-4 w-4" /> Eigenes .vrm hochladen
|
||||
</button>
|
||||
<div className="flex gap-1.5">
|
||||
<input
|
||||
value={urlInput}
|
||||
onChange={(e) => setUrlInput(e.target.value)}
|
||||
onKeyDown={(e) => e.key === "Enter" && loadUrl()}
|
||||
placeholder="…oder VRM-URL einfügen"
|
||||
className="min-w-0 flex-1 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
/>
|
||||
<button onClick={loadUrl} className="rounded-md border border-border/40 bg-background/40 px-2.5 hover:bg-accent" title="Laden">
|
||||
<Link2 className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
<a
|
||||
href="https://hub.vroid.com/en/characters"
|
||||
target="_blank"
|
||||
rel="noopener"
|
||||
className="flex items-center gap-1.5 text-xs text-primary/80 hover:text-primary"
|
||||
>
|
||||
<ExternalLink className="h-3.5 w-3.5" /> Mehr Avatare auf VRoid Hub (kostenlos) → herunterladen & hochladen
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{/* Stimme */}
|
||||
<div>
|
||||
<div className="mb-2 flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wide text-muted-foreground">
|
||||
<Sparkles className="h-3.5 w-3.5" /> Stimme
|
||||
</div>
|
||||
{voices.length === 0 ? (
|
||||
<div className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-300">
|
||||
Voice-Dienst nicht erreichbar — Stimmen werden geladen, sobald der Sidecar läuft.
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<div className="flex gap-1.5">
|
||||
{enginesAvail.map((eng) => (
|
||||
<button
|
||||
key={eng}
|
||||
onClick={() => saveVoice(eng, "")}
|
||||
className={`flex-1 rounded-md border px-2.5 py-1.5 text-xs capitalize transition-colors ${
|
||||
engine === eng ? "border-primary/50 bg-primary/10 text-primary" : "border-border/40 hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
{eng === "piper" ? "Piper (schnell)" : eng === "chatterbox" ? "Chatterbox (premium)" : eng}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<select
|
||||
value={voice}
|
||||
onChange={(e) => saveVoice(engine, e.target.value)}
|
||||
className="w-full rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs outline-none focus:border-primary/50"
|
||||
>
|
||||
<option value="">Standardstimme</option>
|
||||
{voicesForEngine.map((v) => (
|
||||
<option key={v.id} value={v.id}>{v.label}{v.clonable ? " · klonbar" : ""}</option>
|
||||
))}
|
||||
</select>
|
||||
{engine === "chatterbox" && (
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Natürlichste Stimme + Voice-Cloning.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user