Turn-Detection: Smart Turn v3 semantisch (Review P2-11) — live verifiziert
- voice_service /turn: smart-turn-v3.2 (8MB ONNX, ~110ms warm inkl. Features); Audio muss LINKS gepadded werden (rechts-Padding -> konstant 'complete', live diagnostiziert) und der Output ist empirisch P(unfertig) — Doku sagt es andersherum, Messung gewinnt - backend /api/voice/turn: Proxy mit fail-open (Turn-Check ist Optimierung, kein Blocker) - useVAD: Semantik-Hold — bei 'incomplete' bis 1,8s auf Fortsetzung warten und anhaengen, statt mitten im Gedanken zu antworten; Deckel 30s; fail-open bei Netzfehlern - Verifiziert: fertig=true(0.74), mitten-im-Wort=false(0.04), via :9001 ok Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -195,6 +195,76 @@ def transcribe(audio_bytes: bytes, suffix: str, language: str, engine: str = "")
|
||||
return transcribe_whisper(audio_bytes, suffix, language)
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# Turn-Detection — Smart Turn v3 (pipecat, BSD-2): semantisches Äußerungs-Ende.
|
||||
# Whisper-Tiny-Encoder + Klassifikator (8 MB int8, ~12 ms CPU). Der Lucy-Client fragt
|
||||
# nach dem akustischen VAD-Ende hier nach: "War das ein fertiger Satz?" — bei
|
||||
# 'incomplete' wartet er kurz weiter, statt mitten im Gedanken loszuantworten.
|
||||
# Inferenz-Pfad 1:1 aus pipecat-ai/smart-turn inference.py (Feature-Shape muss passen).
|
||||
# =================================================================================
|
||||
TURN_REPO = os.environ.get("VOICE_TURN_REPO", "pipecat-ai/smart-turn-v3")
|
||||
TURN_FILE = os.environ.get("VOICE_TURN_FILE", "smart-turn-v3.2-cpu.onnx")
|
||||
_turn = None # (session, feature_extractor)
|
||||
_turn_failed = False
|
||||
|
||||
|
||||
def turn_model():
|
||||
global _turn, _turn_failed
|
||||
if _turn is None and not _turn_failed:
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
from huggingface_hub import hf_hub_download
|
||||
from transformers import WhisperFeatureExtractor
|
||||
log.info("Lade Smart Turn '%s/%s' …", TURN_REPO, TURN_FILE)
|
||||
path = hf_hub_download(TURN_REPO, TURN_FILE)
|
||||
so = ort.SessionOptions()
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
sess = ort.InferenceSession(path, sess_options=so, providers=["CPUExecutionProvider"])
|
||||
_turn = (sess, WhisperFeatureExtractor(chunk_length=8))
|
||||
except Exception:
|
||||
_turn_failed = True
|
||||
log.exception("Smart Turn nicht verfügbar — /turn meldet complete=true (Fallback).")
|
||||
return _turn
|
||||
|
||||
|
||||
def check_turn(audio_bytes: bytes, suffix: str) -> dict:
|
||||
model = turn_model()
|
||||
if model is None:
|
||||
return {"complete": True, "probability": 1.0, "engine": "none"}
|
||||
sess, fe = model
|
||||
from faster_whisper.audio import decode_audio
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tf:
|
||||
tf.write(audio_bytes)
|
||||
src = tf.name
|
||||
try:
|
||||
import numpy as np
|
||||
arr = decode_audio(src, sampling_rate=16000)
|
||||
# Smart Turn erwartet das Audio AM ENDE des 8-s-Fensters (Zeros vorn) — die Turn-Ende-
|
||||
# Hinweise liegen in den letzten Frames. Der FeatureExtractor padded rechts (Zeros hinten),
|
||||
# damit sah das Modell immer nur Padding und meldete konstant 'complete' (live diagnostiziert).
|
||||
# Darum manuell links auffüllen und exakt 8 s übergeben.
|
||||
n = 8 * 16000
|
||||
arr = arr[-n:]
|
||||
if len(arr) < n:
|
||||
arr = np.concatenate([np.zeros(n - len(arr), dtype=np.float32), arr.astype(np.float32)])
|
||||
inputs = fe(arr, sampling_rate=16000, return_tensors="np", padding="do_not_pad",
|
||||
truncation=True, do_normalize=True)
|
||||
out = sess.run(None, {"input_features": inputs.input_features})
|
||||
# Output-Tensor heißt 'logits' — je nach Export roher Logit ODER schon Sigmoid.
|
||||
# Robust: Werte außerhalb [0,1] durch Sigmoid schicken, sonst direkt nutzen.
|
||||
raw = float(out[0][0].item() if hasattr(out[0][0], "item") else out[0][0])
|
||||
p = raw if 0.0 <= raw <= 1.0 else 1.0 / (1.0 + float(np.exp(-raw)))
|
||||
# EMPIRISCH VERIFIZIERT (2026-07-02, Box): v3.2-cpu liefert P(UNFERTIG) —
|
||||
# fertiger Satz -> 0.26, mitten im Wort abgeschnitten -> 0.96, Stille -> 0.99.
|
||||
# (Die Upstream-Doku beschreibt es andersherum; Messung schlägt Doku.)
|
||||
return {"complete": p <= 0.5, "probability": round(1.0 - p, 3), "engine": "smart-turn-v3"}
|
||||
finally:
|
||||
try:
|
||||
os.unlink(src)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# =================================================================================
|
||||
# TTS — Piper (offizielles Binary; Stimme = ONNX-Datei + .json daneben)
|
||||
# =================================================================================
|
||||
@@ -467,6 +537,16 @@ async def stt(audio: UploadFile = File(...), language: str = Form(default=""),
|
||||
return {"text": text}
|
||||
|
||||
|
||||
@app.post("/turn")
|
||||
async def turn(audio: UploadFile = File(...)) -> dict:
|
||||
"""Semantische Turn-Detection: War die Äußerung ein fertiger Gedanke? (Smart Turn v3)"""
|
||||
data = await audio.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "Leeres Audio.")
|
||||
suffix = Path(audio.filename or "rec.wav").suffix or ".wav"
|
||||
return check_turn(data, suffix)
|
||||
|
||||
|
||||
@app.post("/tts")
|
||||
def tts(body: TTSIn) -> Response:
|
||||
text = (body.text or "").strip()
|
||||
|
||||
Reference in New Issue
Block a user