Hitonabi/mission-control-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: Chatterbox-Klon-Referenz hochladen (eigene Stimme via ElevenLabs → lokal unbegrenzt klonen)

Browse Source 
Sidecar: /reference (POST Upload → 24kHz-Mono-WAV via PyAV/faster-whisper, kein System-ffmpeg;
GET/DELETE). Chatterbox nutzt aktive Referenz (active.txt, überlebt Restart). Backend: /api/voice/
reference-Proxy. Picker (Chatterbox): „Referenz-Stimme hochladen (mp3/wav)" → setzt Stimme auf
»deine Referenz«. Ermöglicht den User-Plan: Stimme besorgen → EL erzeugen → Chatterbox klont.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
2026-06-28 00:55:40 +02:00
parent 75727c6b36
commit eeb397f066
6 changed files with 304 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/routers/voice.py
+36
View File
@@ -79,6 +79,42 @@ async def voice_stt(audio: UploadFile = File(...), language: str = Form(default=
raise HTTPException(502, f"STT fehlgeschlagen: {exc}") raise HTTPException(502, f"STT fehlgeschlagen: {exc}")
@router.post("/voice/reference")
async def voice_set_reference(audio: UploadFile = File(...)) -> dict:
"""Klon-Referenz (z.B. ElevenLabs-Erzeugnis) hochladen → Chatterbox nutzt sie. Proxy → Sidecar."""
data = await audio.read()
if not data:
raise HTTPException(400, "Leeres Audio.")
files = {"audio": (audio.filename or "ref.wav", data, audio.content_type or "audio/mpeg")}
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
r = await client.post(f"{VOICE_SERVICE_URL}/reference", files=files)
r.raise_for_status()
return r.json()
except httpx.HTTPError as exc:
raise HTTPException(502, f"Referenz-Upload fehlgeschlagen: {exc}")
@router.get("/voice/reference")
def voice_get_reference() -> dict:
try:
r = httpx.get(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
r.raise_for_status()
return r.json()
except Exception as exc: # noqa: BLE001
return {"active": False, "error": str(exc)}
@router.delete("/voice/reference")
def voice_clear_reference() -> dict:
try:
r = httpx.delete(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
r.raise_for_status()
return r.json()
except httpx.HTTPError as exc:
raise HTTPException(502, f"Löschen fehlgeschlagen: {exc}")
@router.post("/voice/tts") @router.post("/voice/tts")
async def voice_tts(body: TTSIn) -> Response: async def voice_tts(body: TTSIn) -> Response:
"""Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes.""" """Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes."""
frontend/dist/assets/GraphView-D0BHEwUM.js → frontend/dist/assets/GraphView-BxQ1n8j4.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
frontend/dist/assets/index-DRD104Im.js → frontend/dist/assets/index-Dtig7Izl.js
Vendored
+151 -151
View File
File diff suppressed because one or more lines are too long
frontend/dist/index.html
Vendored
+1 -1
View File
@@ -7,7 +7,7 @@
<link rel="manifest" href="/manifest.webmanifest" /> <link rel="manifest" href="/manifest.webmanifest" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" /> <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<title>Mission Control 2.0</title> <title>Mission Control 2.0</title>
<script type="module" crossorigin src="/assets/index-DRD104Im.js"></script> <script type="module" crossorigin src="/assets/index-Dtig7Izl.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-jNLzkOWh.css"> <link rel="stylesheet" crossorigin href="/assets/index-jNLzkOWh.css">
</head> </head>
<body> <body>
frontend/src/components/voice/AvatarPicker.tsx
+49 -3
View File
@@ -27,9 +27,35 @@ export function AvatarPicker({ avatarUrl, onAvatarChange }: {
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper") const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "") const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
const [previewing, setPreviewing] = useState(false) const [previewing, setPreviewing] = useState(false)
const [refActive, setRefActive] = useState(false)
const [refUploading, setRefUploading] = useState(false)
const fileRef = useRef<HTMLInputElement>(null) const fileRef = useRef<HTMLInputElement>(null)
const refFileRef = useRef<HTMLInputElement>(null)
const previewAudio = useRef<HTMLAudioElement | null>(null) const previewAudio = useRef<HTMLAudioElement | null>(null)
// Aktive Chatterbox-Klon-Referenz anzeigen.
useEffect(() => {
fetch("/api/voice/reference").then((r) => r.json()).then((d) => setRefActive(!!d.active)).catch(() => {})
}, [])
const uploadReference = async (file: File) => {
setRefUploading(true)
try {
const fd = new FormData()
fd.append("audio", file)
const r = await fetch("/api/voice/reference", { method: "POST", body: fd })
if (!r.ok) throw new Error(`${r.status}`)
setRefActive(true)
const v = await fetch("/api/voice/voices").then((r) => r.json())
setVoices(v.voices || [])
saveVoice("chatterbox", "clone") // direkt auf die geklonte Stimme schalten
} catch (e) {
console.error("Referenz-Upload fehlgeschlagen:", e)
} finally {
setRefUploading(false)
}
}
// „Probe hören": synthetisiert einen festen Satz mit der aktuellen Engine+Stimme und spielt ihn ab. // „Probe hören": synthetisiert einen festen Satz mit der aktuellen Engine+Stimme und spielt ihn ab.
const playPreview = async () => { const playPreview = async () => {
if (previewing) return if (previewing) return
@@ -218,9 +244,29 @@ export function AvatarPicker({ avatarUrl, onAvatarChange }: {
</p> </p>
)} )}
{engine === "chatterbox" && ( {engine === "chatterbox" && (
<p className="text-[11px] text-muted-foreground"> <div className="space-y-1.5">
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Lokal + Voice-Cloning (dt. mit Akzent). <input
</p> ref={refFileRef}
type="file"
accept="audio/*,.mp3,.wav,.m4a,.ogg"
className="hidden"
onChange={(e) => { const f = e.target.files?.[0]; if (f) void uploadReference(f) }}
/>
<button
onClick={() => refFileRef.current?.click()}
disabled={refUploading}
className="flex w-full items-center justify-center gap-2 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs hover:bg-accent transition-colors disabled:opacity-60"
>
{refUploading ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Upload className="h-3.5 w-3.5" />}
{refActive ? "Referenz-Stimme ersetzen" : "Referenz-Stimme hochladen (mp3/wav)"}
</button>
<p className="text-[11px] text-muted-foreground">
{refActive
? "✓ Referenz aktiv → wähle oben die Stimme »Chatterbox (deine Referenz)«. "
: "Lad deine (z.B. via ElevenLabs erzeugte) Stimme hoch — Chatterbox klont sie. "}
CPU: paar Sek./Satz. Der Akzent der Referenz überträgt sich.
</p>
</div>
)} )}
{engine === "elevenlabs" && ( {engine === "elevenlabs" && (
<p className="text-[11px] text-muted-foreground"> <p className="text-[11px] text-muted-foreground">
voice_service/app.py
+66 -3
View File
@@ -54,6 +54,20 @@ CHATTERBOX_DEVICE = os.environ.get("VOICE_CHATTERBOX_DEVICE", "cpu")
CHATTERBOX_LANG = os.environ.get("VOICE_CHATTERBOX_LANG", "de") CHATTERBOX_LANG = os.environ.get("VOICE_CHATTERBOX_LANG", "de")
# Optionaler Referenz-WAV für Voice-Cloning (10 s Sprachprobe). Leer = Chatterbox-Standardstimme. # Optionaler Referenz-WAV für Voice-Cloning (10 s Sprachprobe). Leer = Chatterbox-Standardstimme.
CHATTERBOX_REF = os.environ.get("VOICE_CHATTERBOX_REF", "") CHATTERBOX_REF = os.environ.get("VOICE_CHATTERBOX_REF", "")
# Vom Nutzer hochgeladene Klon-Referenz (persistiert über Restarts). active.txt zeigt auf die Datei.
REFS_DIR = Path(os.environ.get("VOICE_REFS_DIR", str(Path.home() / ".voice" / "refs")))
ACTIVE_REF = REFS_DIR / "active.txt"
def active_ref() -> str:
"""Pfad der aktuell aktiven Klon-Referenz (Upload bevorzugt, sonst Env-Default)."""
try:
p = ACTIVE_REF.read_text(encoding="utf-8").strip()
if p and os.path.exists(p):
return p
except OSError:
pass
return CHATTERBOX_REF
# ElevenLabs (Premium-Cloud-TTS): saubere, native deutsche Stimmen (Voice Library). Key wird zur # ElevenLabs (Premium-Cloud-TTS): saubere, native deutsche Stimmen (Voice Library). Key wird zur
# Laufzeit gelesen (env ODER ~/.hermes/.env), damit Nachtragen ohne Code-Deploy reicht. Default- # Laufzeit gelesen (env ODER ~/.hermes/.env), damit Nachtragen ohne Code-Deploy reicht. Default-
# Modell = Flash v2.5 (multilingual, 0,5 Credit/Zeichen → schont das Free-Tier-Kontingent). # Modell = Flash v2.5 (multilingual, 0,5 Credit/Zeichen → schont das Free-Tier-Kontingent).
@@ -166,7 +180,7 @@ def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes:
import soundfile as sf import soundfile as sf
model = chatterbox_model() model = chatterbox_model()
kwargs = {"language_id": language or CHATTERBOX_LANG} kwargs = {"language_id": language or CHATTERBOX_LANG}
ref = ref_path or CHATTERBOX_REF ref = ref_path or active_ref()
if ref and os.path.exists(ref): if ref and os.path.exists(ref):
kwargs["audio_prompt_path"] = ref # Zero-Shot Voice-Cloning aus Referenz kwargs["audio_prompt_path"] = ref # Zero-Shot Voice-Cloning aus Referenz
wav = model.generate(text, **kwargs) wav = model.generate(text, **kwargs)
@@ -181,8 +195,8 @@ def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes:
def chatterbox_list() -> list[dict]: def chatterbox_list() -> list[dict]:
# Chatterbox hat keine festen „Stimm-Dateien": Standardstimme + optionale Klon-Referenz. # Chatterbox hat keine festen „Stimm-Dateien": Standardstimme + optionale Klon-Referenz.
items = [{"engine": "chatterbox", "id": "default", "label": "Chatterbox (Standard, dt.)", "clonable": True}] items = [{"engine": "chatterbox", "id": "default", "label": "Chatterbox (Standard, dt.)", "clonable": True}]
if CHATTERBOX_REF and os.path.exists(CHATTERBOX_REF): if active_ref():
items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (geklonte Stimme)", "clonable": True}) items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (deine Referenz)", "clonable": True})
return items return items
@@ -349,6 +363,55 @@ def tts(body: TTSIn) -> Response:
return Response(content=piper_tts(text, body.voice), media_type="audio/wav") return Response(content=piper_tts(text, body.voice), media_type="audio/wav")
def _save_reference_as_wav(data: bytes, filename: str) -> str:
"""Upload (mp3/wav/…) → 24 kHz Mono-WAV für Chatterbox. Dekodierung via PyAV (steckt in
faster-whisper → kein System-ffmpeg nötig); Fallback soundfile, sonst Rohdatei."""
REFS_DIR.mkdir(parents=True, exist_ok=True)
raw = REFS_DIR / ("upload" + (Path(filename or "ref").suffix or ".bin"))
raw.write_bytes(data)
dest = REFS_DIR / "ref.wav"
import soundfile as sf
try:
from faster_whisper.audio import decode_audio # PyAV-basiert, kann mp3/m4a/ogg/wav
arr = decode_audio(str(raw), sampling_rate=24000)
sf.write(str(dest), arr, 24000, subtype="PCM_16")
return str(dest)
except Exception:
log.warning("PyAV-Dekodierung fehlgeschlagen, versuche soundfile", exc_info=True)
try:
arr, srr = sf.read(str(raw))
sf.write(str(dest), arr, srr, subtype="PCM_16")
return str(dest)
except Exception:
return str(raw) # Fallback: Chatterbox versucht selbst zu laden
@app.post("/reference")
async def set_reference(audio: UploadFile = File(...)) -> dict:
"""Klon-Referenz hochladen (z.B. ElevenLabs-Erzeugnis) → Chatterbox nutzt sie als Stimme."""
data = await audio.read()
if not data:
raise HTTPException(400, "Leeres Audio.")
path = _save_reference_as_wav(data, audio.filename or "ref.wav")
ACTIVE_REF.write_text(path, encoding="utf-8")
return {"ok": True, "path": path, "bytes": len(data)}
@app.get("/reference")
def get_reference() -> dict:
p = active_ref()
return {"active": bool(p), "path": p}
@app.delete("/reference")
def clear_reference() -> dict:
try:
ACTIVE_REF.unlink(missing_ok=True)
except OSError:
pass
return {"ok": True}
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn