Feat: Chatterbox-Klon-Referenz hochladen (eigene Stimme via ElevenLabs → lokal unbegrenzt klonen)
Sidecar: /reference (POST Upload → 24kHz-Mono-WAV via PyAV/faster-whisper, kein System-ffmpeg; GET/DELETE). Chatterbox nutzt aktive Referenz (active.txt, überlebt Restart). Backend: /api/voice/ reference-Proxy. Picker (Chatterbox): „Referenz-Stimme hochladen (mp3/wav)" → setzt Stimme auf »deine Referenz«. Ermöglicht den User-Plan: Stimme besorgen → EL erzeugen → Chatterbox klont. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -79,6 +79,42 @@ async def voice_stt(audio: UploadFile = File(...), language: str = Form(default=
|
||||
raise HTTPException(502, f"STT fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/reference")
|
||||
async def voice_set_reference(audio: UploadFile = File(...)) -> dict:
|
||||
"""Klon-Referenz (z.B. ElevenLabs-Erzeugnis) hochladen → Chatterbox nutzt sie. Proxy → Sidecar."""
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
files = {"audio": (audio.filename or "ref.wav", data, audio.content_type or "audio/mpeg")}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(f"{VOICE_SERVICE_URL}/reference", files=files)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"Referenz-Upload fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.get("/voice/reference")
|
||||
def voice_get_reference() -> dict:
|
||||
try:
|
||||
r = httpx.get(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"active": False, "error": str(exc)}
|
||||
|
||||
|
||||
@router.delete("/voice/reference")
|
||||
def voice_clear_reference() -> dict:
|
||||
try:
|
||||
r = httpx.delete(f"{VOICE_SERVICE_URL}/reference", timeout=httpx.Timeout(8.0))
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except httpx.HTTPError as exc:
|
||||
raise HTTPException(502, f"Löschen fehlgeschlagen: {exc}")
|
||||
|
||||
|
||||
@router.post("/voice/tts")
|
||||
async def voice_tts(body: TTSIn) -> Response:
|
||||
"""Text → Sprache (Proxy auf Sidecar /tts), liefert WAV-Bytes."""
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+151
-151
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
||||
<link rel="manifest" href="/manifest.webmanifest" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<title>Mission Control 2.0</title>
|
||||
<script type="module" crossorigin src="/assets/index-DRD104Im.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-Dtig7Izl.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-jNLzkOWh.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -27,9 +27,35 @@ export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
const [engine, setEngine] = useState(localStorage.getItem("mc_voice_engine") || "piper")
|
||||
const [voice, setVoice] = useState(localStorage.getItem("mc_voice_voice") || "")
|
||||
const [previewing, setPreviewing] = useState(false)
|
||||
const [refActive, setRefActive] = useState(false)
|
||||
const [refUploading, setRefUploading] = useState(false)
|
||||
const fileRef = useRef<HTMLInputElement>(null)
|
||||
const refFileRef = useRef<HTMLInputElement>(null)
|
||||
const previewAudio = useRef<HTMLAudioElement | null>(null)
|
||||
|
||||
// Aktive Chatterbox-Klon-Referenz anzeigen.
|
||||
useEffect(() => {
|
||||
fetch("/api/voice/reference").then((r) => r.json()).then((d) => setRefActive(!!d.active)).catch(() => {})
|
||||
}, [])
|
||||
|
||||
const uploadReference = async (file: File) => {
|
||||
setRefUploading(true)
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", file)
|
||||
const r = await fetch("/api/voice/reference", { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`${r.status}`)
|
||||
setRefActive(true)
|
||||
const v = await fetch("/api/voice/voices").then((r) => r.json())
|
||||
setVoices(v.voices || [])
|
||||
saveVoice("chatterbox", "clone") // direkt auf die geklonte Stimme schalten
|
||||
} catch (e) {
|
||||
console.error("Referenz-Upload fehlgeschlagen:", e)
|
||||
} finally {
|
||||
setRefUploading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// „Probe hören": synthetisiert einen festen Satz mit der aktuellen Engine+Stimme und spielt ihn ab.
|
||||
const playPreview = async () => {
|
||||
if (previewing) return
|
||||
@@ -218,9 +244,29 @@ export function AvatarPicker({ avatarUrl, onAvatarChange }: {
|
||||
</p>
|
||||
)}
|
||||
{engine === "chatterbox" && (
|
||||
<div className="space-y-1.5">
|
||||
<input
|
||||
ref={refFileRef}
|
||||
type="file"
|
||||
accept="audio/*,.mp3,.wav,.m4a,.ogg"
|
||||
className="hidden"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) void uploadReference(f) }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => refFileRef.current?.click()}
|
||||
disabled={refUploading}
|
||||
className="flex w-full items-center justify-center gap-2 rounded-md border border-border/40 bg-background/40 px-2.5 py-2 text-xs hover:bg-accent transition-colors disabled:opacity-60"
|
||||
>
|
||||
{refUploading ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Upload className="h-3.5 w-3.5" />}
|
||||
{refActive ? "Referenz-Stimme ersetzen" : "Referenz-Stimme hochladen (mp3/wav)"}
|
||||
</button>
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
Chatterbox läuft auf CPU → erste Antwort kann ein paar Sekunden dauern. Lokal + Voice-Cloning (dt. mit Akzent).
|
||||
{refActive
|
||||
? "✓ Referenz aktiv → wähle oben die Stimme »Chatterbox (deine Referenz)«. "
|
||||
: "Lad deine (z.B. via ElevenLabs erzeugte) Stimme hoch — Chatterbox klont sie. "}
|
||||
CPU: paar Sek./Satz. Der Akzent der Referenz überträgt sich.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{engine === "elevenlabs" && (
|
||||
<p className="text-[11px] text-muted-foreground">
|
||||
|
||||
+66
-3
@@ -54,6 +54,20 @@ CHATTERBOX_DEVICE = os.environ.get("VOICE_CHATTERBOX_DEVICE", "cpu")
|
||||
CHATTERBOX_LANG = os.environ.get("VOICE_CHATTERBOX_LANG", "de")
|
||||
# Optionaler Referenz-WAV für Voice-Cloning (10 s Sprachprobe). Leer = Chatterbox-Standardstimme.
|
||||
CHATTERBOX_REF = os.environ.get("VOICE_CHATTERBOX_REF", "")
|
||||
# Vom Nutzer hochgeladene Klon-Referenz (persistiert über Restarts). active.txt zeigt auf die Datei.
|
||||
REFS_DIR = Path(os.environ.get("VOICE_REFS_DIR", str(Path.home() / ".voice" / "refs")))
|
||||
ACTIVE_REF = REFS_DIR / "active.txt"
|
||||
|
||||
|
||||
def active_ref() -> str:
|
||||
"""Pfad der aktuell aktiven Klon-Referenz (Upload bevorzugt, sonst Env-Default)."""
|
||||
try:
|
||||
p = ACTIVE_REF.read_text(encoding="utf-8").strip()
|
||||
if p and os.path.exists(p):
|
||||
return p
|
||||
except OSError:
|
||||
pass
|
||||
return CHATTERBOX_REF
|
||||
# ElevenLabs (Premium-Cloud-TTS): saubere, native deutsche Stimmen (Voice Library). Key wird zur
|
||||
# Laufzeit gelesen (env ODER ~/.hermes/.env), damit Nachtragen ohne Code-Deploy reicht. Default-
|
||||
# Modell = Flash v2.5 (multilingual, 0,5 Credit/Zeichen → schont das Free-Tier-Kontingent).
|
||||
@@ -166,7 +180,7 @@ def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes:
|
||||
import soundfile as sf
|
||||
model = chatterbox_model()
|
||||
kwargs = {"language_id": language or CHATTERBOX_LANG}
|
||||
ref = ref_path or CHATTERBOX_REF
|
||||
ref = ref_path or active_ref()
|
||||
if ref and os.path.exists(ref):
|
||||
kwargs["audio_prompt_path"] = ref # Zero-Shot Voice-Cloning aus Referenz
|
||||
wav = model.generate(text, **kwargs)
|
||||
@@ -181,8 +195,8 @@ def chatterbox_tts(text: str, language: str, ref_path: str) -> bytes:
|
||||
def chatterbox_list() -> list[dict]:
|
||||
# Chatterbox hat keine festen „Stimm-Dateien": Standardstimme + optionale Klon-Referenz.
|
||||
items = [{"engine": "chatterbox", "id": "default", "label": "Chatterbox (Standard, dt.)", "clonable": True}]
|
||||
if CHATTERBOX_REF and os.path.exists(CHATTERBOX_REF):
|
||||
items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (geklonte Stimme)", "clonable": True})
|
||||
if active_ref():
|
||||
items.append({"engine": "chatterbox", "id": "clone", "label": "Chatterbox (deine Referenz)", "clonable": True})
|
||||
return items
|
||||
|
||||
|
||||
@@ -349,6 +363,55 @@ def tts(body: TTSIn) -> Response:
|
||||
return Response(content=piper_tts(text, body.voice), media_type="audio/wav")
|
||||
|
||||
|
||||
def _save_reference_as_wav(data: bytes, filename: str) -> str:
|
||||
"""Upload (mp3/wav/…) → 24 kHz Mono-WAV für Chatterbox. Dekodierung via PyAV (steckt in
|
||||
faster-whisper → kein System-ffmpeg nötig); Fallback soundfile, sonst Rohdatei."""
|
||||
REFS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
raw = REFS_DIR / ("upload" + (Path(filename or "ref").suffix or ".bin"))
|
||||
raw.write_bytes(data)
|
||||
dest = REFS_DIR / "ref.wav"
|
||||
import soundfile as sf
|
||||
try:
|
||||
from faster_whisper.audio import decode_audio # PyAV-basiert, kann mp3/m4a/ogg/wav
|
||||
arr = decode_audio(str(raw), sampling_rate=24000)
|
||||
sf.write(str(dest), arr, 24000, subtype="PCM_16")
|
||||
return str(dest)
|
||||
except Exception:
|
||||
log.warning("PyAV-Dekodierung fehlgeschlagen, versuche soundfile", exc_info=True)
|
||||
try:
|
||||
arr, srr = sf.read(str(raw))
|
||||
sf.write(str(dest), arr, srr, subtype="PCM_16")
|
||||
return str(dest)
|
||||
except Exception:
|
||||
return str(raw) # Fallback: Chatterbox versucht selbst zu laden
|
||||
|
||||
|
||||
@app.post("/reference")
|
||||
async def set_reference(audio: UploadFile = File(...)) -> dict:
|
||||
"""Klon-Referenz hochladen (z.B. ElevenLabs-Erzeugnis) → Chatterbox nutzt sie als Stimme."""
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
path = _save_reference_as_wav(data, audio.filename or "ref.wav")
|
||||
ACTIVE_REF.write_text(path, encoding="utf-8")
|
||||
return {"ok": True, "path": path, "bytes": len(data)}
|
||||
|
||||
|
||||
@app.get("/reference")
|
||||
def get_reference() -> dict:
|
||||
p = active_ref()
|
||||
return {"active": bool(p), "path": p}
|
||||
|
||||
|
||||
@app.delete("/reference")
|
||||
def clear_reference() -> dict:
|
||||
try:
|
||||
ACTIVE_REF.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
|
||||
Reference in New Issue
Block a user