Lucy ausgegliedert: client/lucy-* -> eigenes Repo Hitonabi/lucy (+ alte hermes-voice geloescht)
- lucy-desktop + lucy-tts leben jetzt in F:/Coding Stuff/lucy bzw. git .../Hitonabi/lucy (Verzeichnisnamen unveraendert -> alle Pfade/BATs funktionieren weiter) - lucy-f5 geloescht (Experiment abgeschlossen, Verdikt: Pocket bleibt; Skripte in der Historie) - hermes-voice geloescht (Vor-Lucy-Python-Client, ersetzt) - docs/LUCY_TTS_PLAN.md mit umgezogen; .gitignore entschlackt - MC2 = Box/Stack (Backend, Frontend, Sidecars, Deploy, hermes-pc-Executor) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
-22
@@ -20,25 +20,3 @@ frontend/dist/avatar.vrm
|
||||
box_recon*
|
||||
gemma_swap*
|
||||
|
||||
# Lucy-TTS/F5-Experimente: nur Code/Batch committen — venvs, Modelle, Audio, Logs bleiben lokal
|
||||
client/lucy-tts/ptts-venv/
|
||||
client/lucy-tts/venv/
|
||||
client/lucy-tts/llamacpp-vulkan/
|
||||
client/lucy-tts/llamacpp-vulkan.zip
|
||||
client/lucy-tts/models/
|
||||
client/lucy-tts/out_*/
|
||||
client/lucy-tts/*.wav
|
||||
client/lucy-tts/*.mp3
|
||||
client/lucy-tts/*.log
|
||||
client/lucy-tts/*.safetensors
|
||||
client/lucy-tts/*.json
|
||||
client/lucy-f5/.cache/
|
||||
client/lucy-f5/F5-TTS-ONNX/
|
||||
client/lucy-f5/onnx_de/
|
||||
client/lucy-f5/onnx_f32/
|
||||
client/lucy-f5/vocos-mel-24khz/
|
||||
client/lucy-f5/*.onnx
|
||||
client/lucy-f5/*.safetensors
|
||||
client/lucy-f5/*.wav
|
||||
client/lucy-f5/*.zip
|
||||
client/lucy-f5/venv/
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
"""Spricht mit dem Hermes Agent Daemon auf der AI Box."""
|
||||
import httpx
|
||||
|
||||
TIMEOUT = 120
|
||||
|
||||
|
||||
def ask_agent(text: str, screenshot_b64: str | None, settings: dict) -> tuple[str, list]:
|
||||
"""Schickt Nachricht an den Daemon, gibt (response, tool_calls) zurück."""
|
||||
daemon_url = settings.get("agent_daemon_url", "http://192.168.178.151:8765")
|
||||
try:
|
||||
with httpx.Client(timeout=TIMEOUT) as c:
|
||||
resp = c.post(f"{daemon_url}/chat", json={
|
||||
"message": text,
|
||||
"screenshot": screenshot_b64,
|
||||
})
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data.get("response", ""), data.get("tool_calls", [])
|
||||
except httpx.TimeoutException:
|
||||
return "Antwort hat zu lange gedauert.", []
|
||||
except Exception as e:
|
||||
return f"Agent nicht erreichbar: {e}", []
|
||||
|
||||
|
||||
def get_agent_status(settings: dict) -> dict:
|
||||
daemon_url = settings.get("agent_daemon_url", "http://192.168.178.151:8765")
|
||||
try:
|
||||
with httpx.Client(timeout=5) as c:
|
||||
return c.get(f"{daemon_url}/status").json()
|
||||
except Exception:
|
||||
return {"alive": False}
|
||||
@@ -1,47 +0,0 @@
|
||||
import httpx
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"Du bist Hermes, ein hilfreicher KI-Assistent. "
|
||||
"Antworte kurz und präzise, da deine Antwort vorgelesen wird. "
|
||||
"Maximal 3-4 Sätze, kein Markdown."
|
||||
)
|
||||
REQUEST_TIMEOUT = 30
|
||||
|
||||
|
||||
def ask(text: str, screenshot_b64: str | None, settings: dict) -> str:
|
||||
"""Schickt Text (+ optionalen Screenshot) an MC2 und gibt die Antwort zurück."""
|
||||
use_vision = screenshot_b64 is not None
|
||||
model = settings.get("vision_model", "Qwen3-VL-2B-Instruct") if use_vision \
|
||||
else settings.get("chat_model", "Hermes-4-14B")
|
||||
base_url = settings.get("mc2_url", "http://192.168.178.151:9001/v1")
|
||||
max_tokens = int(settings.get("max_response_tokens", 200))
|
||||
|
||||
if use_vision:
|
||||
user_content = [
|
||||
{"type": "text", "text": text},
|
||||
{"type": "image_url", "image_url": {
|
||||
"url": f"data:image/jpeg;base64,{screenshot_b64}"
|
||||
}},
|
||||
]
|
||||
else:
|
||||
user_content = text
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_content},
|
||||
],
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
|
||||
response = client.post(f"{base_url}/chat/completions", json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()["choices"][0]["message"]["content"].strip()
|
||||
except httpx.TimeoutException:
|
||||
return "Entschuldigung, die Antwort hat zu lange gedauert."
|
||||
except Exception as e:
|
||||
return f"Verbindungsfehler: {e}"
|
||||
@@ -1,91 +0,0 @@
|
||||
"""
|
||||
Audio-Input: Aufnahme + Whisper STT.
|
||||
Kein Wake-Word-Loop — Aufnahme startet direkt auf Hotkey-Signal.
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import scipy.io.wavfile as wav
|
||||
from faster_whisper import WhisperModel
|
||||
from pathlib import Path
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
CHUNK = 1024
|
||||
ENERGY_THRESHOLD = 0.008
|
||||
|
||||
_model: WhisperModel | None = None
|
||||
_model_size: str = ""
|
||||
_recording = False
|
||||
_frames: list[np.ndarray] = []
|
||||
_stream: sd.InputStream | None = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def load_model(model_size: str):
|
||||
global _model, _model_size
|
||||
if _model is None or _model_size != model_size:
|
||||
model_dir = Path.home() / ".hermes-voice" / "whisper-models"
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
_model = WhisperModel(
|
||||
model_size,
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
download_root=str(model_dir),
|
||||
)
|
||||
_model_size = model_size
|
||||
|
||||
|
||||
def start_recording():
|
||||
global _recording, _frames, _stream
|
||||
with _lock:
|
||||
_recording = True
|
||||
_frames = []
|
||||
|
||||
def callback(indata, frames, time, status):
|
||||
if _recording:
|
||||
_frames.append(indata[:, 0].copy())
|
||||
|
||||
_stream = sd.InputStream(
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1,
|
||||
dtype="float32",
|
||||
blocksize=CHUNK,
|
||||
callback=callback,
|
||||
)
|
||||
_stream.start()
|
||||
|
||||
|
||||
def stop_recording() -> np.ndarray:
|
||||
global _recording, _stream
|
||||
with _lock:
|
||||
_recording = False
|
||||
if _stream:
|
||||
_stream.stop()
|
||||
_stream.close()
|
||||
_stream = None
|
||||
if not _frames:
|
||||
return np.array([], dtype=np.float32)
|
||||
return np.concatenate(_frames)
|
||||
|
||||
|
||||
def transcribe(audio: np.ndarray, model_size: str, language: str) -> str:
|
||||
if len(audio) < SAMPLE_RATE * 0.3:
|
||||
return ""
|
||||
load_model(model_size)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
tmp = f.name
|
||||
try:
|
||||
wav.write(tmp, SAMPLE_RATE, (audio * 32767).astype(np.int16))
|
||||
lang = language if language != "auto" else None
|
||||
segments, _ = _model.transcribe(
|
||||
tmp,
|
||||
language=lang,
|
||||
beam_size=3,
|
||||
vad_filter=True,
|
||||
)
|
||||
return " ".join(s.text for s in segments).strip()
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
@@ -1,70 +0,0 @@
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import pygame
|
||||
import edge_tts
|
||||
|
||||
pygame.mixer.init()
|
||||
_stop_event = threading.Event()
|
||||
_speak_lock = threading.Lock()
|
||||
|
||||
|
||||
def stop_speaking():
|
||||
_stop_event.set()
|
||||
pygame.mixer.music.stop()
|
||||
|
||||
|
||||
def speak(text: str, voice: str = "de-DE-SeraphinaMultilingualNeural", rate: str = "+0%"):
|
||||
"""Text → Edge TTS → Lautsprecher. Blockiert bis fertig oder unterbrochen."""
|
||||
with _speak_lock:
|
||||
_stop_event.clear()
|
||||
|
||||
tmp_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
|
||||
tmp_path = f.name
|
||||
|
||||
# Edge TTS in eigenem Event-Loop (Thread-sicher)
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
communicate = edge_tts.Communicate(text, voice, rate=rate)
|
||||
loop.run_until_complete(communicate.save(tmp_path))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
if _stop_event.is_set():
|
||||
return
|
||||
|
||||
pygame.mixer.music.load(tmp_path)
|
||||
pygame.mixer.music.play()
|
||||
|
||||
# Polling ohne pygame.time.wait (blockiert Event-Loop nicht)
|
||||
while pygame.mixer.music.get_busy():
|
||||
if _stop_event.is_set():
|
||||
pygame.mixer.music.stop()
|
||||
break
|
||||
time.sleep(0.05)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def play_ding():
|
||||
"""Kurzer Aktivierungs-Ton (440 Hz, 120ms)."""
|
||||
import numpy as np
|
||||
sample_rate = 44100
|
||||
duration = 0.12
|
||||
t = np.linspace(0, duration, int(sample_rate * duration), False)
|
||||
wave = (np.sin(2 * np.pi * 440 * t) * 0.3 * 32767).astype(np.int16)
|
||||
stereo = np.column_stack([wave, wave])
|
||||
sound = pygame.sndarray.make_sound(stereo)
|
||||
sound.play()
|
||||
time.sleep(duration + 0.02)
|
||||
@@ -1,46 +0,0 @@
|
||||
@echo off
|
||||
cd /d "%~dp0"
|
||||
|
||||
echo ========================================
|
||||
echo Hermes Voice Client -- Build (.exe)
|
||||
echo ========================================
|
||||
echo.
|
||||
|
||||
:: Venv pruefen
|
||||
if not exist ".venv\Scripts\activate.bat" (
|
||||
echo [FEHLER] Bitte erst setup.bat ausfuehren.
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
:: PyInstaller installieren falls noetig
|
||||
.venv\Scripts\pip install pyinstaller -q
|
||||
|
||||
echo [1/2] Baue HermesVoice.exe ...
|
||||
.venv\Scripts\pyinstaller ^
|
||||
--onefile ^
|
||||
--windowed ^
|
||||
--name HermesVoice ^
|
||||
--add-data "assets;assets" ^
|
||||
--hidden-import "customtkinter" ^
|
||||
--hidden-import "pynput.keyboard._win32" ^
|
||||
--hidden-import "pynput.mouse._win32" ^
|
||||
--collect-all "customtkinter" ^
|
||||
main.py
|
||||
|
||||
echo.
|
||||
if exist "dist\HermesVoice.exe" (
|
||||
echo [2/2] Fertig!
|
||||
echo.
|
||||
echo dist\HermesVoice.exe ^(%.0f MB^)
|
||||
echo.
|
||||
echo Die .exe benoetigt beim ersten Start Internet fuer:
|
||||
echo - Whisper-Modell-Download ^(~150MB fuer "small"^)
|
||||
echo - Edge TTS ^(online^)
|
||||
echo.
|
||||
for %%I in ("dist\HermesVoice.exe") do echo Groesse: %%~zI Bytes
|
||||
) else (
|
||||
echo [FEHLER] Build fehlgeschlagen. Siehe build-Log oben.
|
||||
)
|
||||
echo ========================================
|
||||
pause
|
||||
@@ -1,62 +0,0 @@
|
||||
# Hermes Voice Client — Konfiguration
|
||||
# Alle Einstellungen hier anpassen, kein Code-Edit nötig.
|
||||
|
||||
# ── AI-Box ──────────────────────────────────────────────────────────────
|
||||
MC2_BASE_URL = "http://192.168.178.151:9001/v1"
|
||||
|
||||
# Modell für reine Text-Anfragen (kein Screenshot)
|
||||
CHAT_MODEL = "Hermes-4-14B"
|
||||
|
||||
# Modell wenn ein Screenshot mitgeschickt wird
|
||||
VISION_MODEL = "Qwen3-VL-2B-Instruct"
|
||||
|
||||
# ── Wake Word ────────────────────────────────────────────────────────────
|
||||
# Einfach den gewünschten Trigger-Text hier eintragen — kein Account, kein Download.
|
||||
# Whisper transkribiert Sprache und prüft ob eines der Wörter enthalten ist.
|
||||
# Mehrere Varianten möglich: ["hey hermes", "hermes", "hey jarvis"]
|
||||
WAKE_WORDS = ["hey hermes", "hermes"]
|
||||
|
||||
# Whisper-Modell für die schnelle Wake-Detection (tiny = 39MB, sehr schnell)
|
||||
WAKE_WHISPER_MODEL = "tiny"
|
||||
|
||||
# ── Spracheingabe ────────────────────────────────────────────────────────
|
||||
# faster-whisper Modellgröße: "tiny", "base", "small", "medium"
|
||||
# "small" läuft gut auf CPU (~244 MB), "base" ist schneller aber ungenauer
|
||||
WHISPER_MODEL_SIZE = "small"
|
||||
WHISPER_LANGUAGE = "de" # "de" für Deutsch, "en" für Englisch, None = auto
|
||||
|
||||
# Sekunden Stille bis Aufnahme endet
|
||||
SILENCE_TIMEOUT = 1.5
|
||||
# Maximale Aufnahmedauer in Sekunden (Sicherheitsnetz)
|
||||
MAX_RECORD_SECONDS = 20
|
||||
|
||||
# ── Sprachausgabe ────────────────────────────────────────────────────────
|
||||
# Edge TTS Stimmen: https://speech.microsoft.com/portal/voicegallery
|
||||
# Deutsch: "de-DE-KillianNeural", "de-DE-SeraphinaMultilingualNeural"
|
||||
# Englisch: "en-US-AndrewNeural", "en-US-AriaNeural"
|
||||
TTS_VOICE = "de-DE-KillianNeural"
|
||||
TTS_RATE = "+0%" # Geschwindigkeit: "+10%" schneller, "-10%" langsamer
|
||||
TTS_PITCH = "+0Hz" # Tonhöhe
|
||||
|
||||
# ── Screen Capture ───────────────────────────────────────────────────────
|
||||
# Screenshot bei jeder Anfrage mitschicken?
|
||||
SCREENSHOT_ON_QUERY = True
|
||||
# Monitor-Index (0 = alle, 1 = primär, 2 = zweiter Monitor)
|
||||
SCREENSHOT_MONITOR = 1
|
||||
# Auflösung für Screenshot (kleinere = schnellere Übertragung)
|
||||
SCREENSHOT_WIDTH = 1280
|
||||
SCREENSHOT_HEIGHT = 720
|
||||
|
||||
# ── Agent-Verhalten ──────────────────────────────────────────────────────
|
||||
MAX_RESPONSE_TOKENS = 200 # kurze gesprochene Antworten
|
||||
REQUEST_TIMEOUT = 30 # Sekunden bis Timeout
|
||||
|
||||
SYSTEM_PROMPT = """Du bist Hermes, ein KI-Assistent der direkt in den lokalen AI-Homelab integriert ist.
|
||||
Du hörst die Stimme des Nutzers und siehst seinen Bildschirm.
|
||||
|
||||
Regeln für Antworten:
|
||||
- Kurz und präzise (2–4 Sätze maximum)
|
||||
- Kein Markdown, keine Aufzählungen, keine Codeblöcke — du wirst gesprochen
|
||||
- Wenn du einen offensichtlichen Fehler auf dem Bildschirm siehst, weise kurz darauf hin
|
||||
- Antworte auf Deutsch wenn der Nutzer Deutsch spricht, auf Englisch wenn Englisch
|
||||
- Sei direkt und hilfreich, kein unnötiges Smalltalk"""
|
||||
@@ -1,39 +0,0 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
CONFIG_DIR = Path.home() / ".hermes-voice"
|
||||
SETTINGS_FILE = CONFIG_DIR / "settings.json"
|
||||
|
||||
DEFAULTS = {
|
||||
"mc2_url": "http://192.168.178.151:9001/v1",
|
||||
"chat_model": "Hermes-4-14B",
|
||||
"vision_model": "Qwen3-VL-2B-Instruct",
|
||||
"hotkey": "ctrl_r",
|
||||
"tts_voice": "de-DE-SeraphinaMultilingualNeural",
|
||||
"tts_rate": "+0%",
|
||||
"whisper_model": "small",
|
||||
"language": "de",
|
||||
"screenshot_enabled": True,
|
||||
"screenshot_monitor": 1,
|
||||
"max_response_tokens": 200,
|
||||
"silence_timeout": 1.5,
|
||||
}
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
CONFIG_DIR.mkdir(exist_ok=True)
|
||||
if SETTINGS_FILE.exists():
|
||||
try:
|
||||
saved = json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
|
||||
return {**DEFAULTS, **saved}
|
||||
except Exception:
|
||||
pass
|
||||
return dict(DEFAULTS)
|
||||
|
||||
|
||||
def save(settings: dict):
|
||||
CONFIG_DIR.mkdir(exist_ok=True)
|
||||
SETTINGS_FILE.write_text(
|
||||
json.dumps(settings, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Globaler Push-to-Talk Hotkey via pynput.
|
||||
Funktioniert auch wenn das Fenster minimiert/im Hintergrund ist.
|
||||
"""
|
||||
import threading
|
||||
from pynput import keyboard
|
||||
|
||||
# Mapping: config-String → pynput Key/KeyCode
|
||||
_SPECIAL = {
|
||||
"ctrl_r": keyboard.Key.ctrl_r,
|
||||
"ctrl_l": keyboard.Key.ctrl_l,
|
||||
"alt_r": keyboard.Key.alt_r,
|
||||
"alt_l": keyboard.Key.alt_l,
|
||||
"shift_r": keyboard.Key.shift_r,
|
||||
"shift_l": keyboard.Key.shift_l,
|
||||
"f13": keyboard.Key.f13,
|
||||
"f14": keyboard.Key.f14,
|
||||
"f15": keyboard.Key.f15,
|
||||
"f16": keyboard.Key.f16,
|
||||
"caps_lock": keyboard.Key.caps_lock,
|
||||
"scroll_lock": keyboard.Key.scroll_lock,
|
||||
"pause": keyboard.Key.pause,
|
||||
}
|
||||
|
||||
DISPLAY_NAMES = {
|
||||
"ctrl_r": "Rechte Strg",
|
||||
"ctrl_l": "Linke Strg",
|
||||
"alt_r": "Rechte Alt",
|
||||
"alt_l": "Linke Alt",
|
||||
"shift_r": "Rechte Shift",
|
||||
"shift_l": "Linke Shift",
|
||||
"f13": "F13", "f14": "F14", "f15": "F15", "f16": "F16",
|
||||
"caps_lock": "Caps Lock",
|
||||
"scroll_lock": "Scroll Lock",
|
||||
"pause": "Pause",
|
||||
}
|
||||
|
||||
|
||||
def key_to_str(key) -> str:
|
||||
"""Konvertiert pynput Key → config-String."""
|
||||
for name, k in _SPECIAL.items():
|
||||
if key == k:
|
||||
return name
|
||||
if hasattr(key, "char") and key.char:
|
||||
return key.char.lower()
|
||||
return str(key).replace("Key.", "")
|
||||
|
||||
|
||||
def str_to_display(key_str: str) -> str:
|
||||
return DISPLAY_NAMES.get(key_str, key_str.upper())
|
||||
|
||||
|
||||
class HotkeyListener:
|
||||
def __init__(self, key_str: str, on_press_cb, on_release_cb):
|
||||
self._key_str = key_str
|
||||
self._on_press = on_press_cb
|
||||
self._on_release = on_release_cb
|
||||
self._pressed = False
|
||||
self._listener: keyboard.Listener | None = None
|
||||
|
||||
def _target_key(self):
|
||||
return _SPECIAL.get(self._key_str) or keyboard.KeyCode.from_char(self._key_str)
|
||||
|
||||
def _on_press_raw(self, key):
|
||||
if not self._pressed and key == self._target_key():
|
||||
self._pressed = True
|
||||
self._on_press()
|
||||
|
||||
def _on_release_raw(self, key):
|
||||
if self._pressed and key == self._target_key():
|
||||
self._pressed = False
|
||||
self._on_release()
|
||||
|
||||
def start(self):
|
||||
self._listener = keyboard.Listener(
|
||||
on_press=self._on_press_raw,
|
||||
on_release=self._on_release_raw,
|
||||
)
|
||||
self._listener.start()
|
||||
|
||||
def stop(self):
|
||||
if self._listener:
|
||||
self._listener.stop()
|
||||
|
||||
def update_key(self, key_str: str):
|
||||
self._key_str = key_str
|
||||
self._pressed = False
|
||||
|
||||
|
||||
class KeyCapturer:
|
||||
"""Einmalig den nächsten Tastendruck abfangen für Hotkey-Konfiguration."""
|
||||
|
||||
def __init__(self, callback):
|
||||
self._callback = callback
|
||||
self._listener: keyboard.Listener | None = None
|
||||
|
||||
def start(self):
|
||||
def on_press(key):
|
||||
key_str = key_to_str(key)
|
||||
self._listener.stop()
|
||||
self._callback(key_str)
|
||||
self._listener = keyboard.Listener(on_press=on_press)
|
||||
self._listener.start()
|
||||
@@ -1,325 +0,0 @@
|
||||
"""
|
||||
Hermes Voice Client — GUI
|
||||
Push-to-Talk: Hotkey halten → sprechen → loslassen → Hermes antwortet
|
||||
"""
|
||||
import queue
|
||||
import threading
|
||||
import sys
|
||||
import customtkinter as ctk
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
import config_manager as cfg
|
||||
from audio_input import start_recording, stop_recording, transcribe
|
||||
from audio_output import speak, stop_speaking, play_ding
|
||||
from screen_capture import capture_screen
|
||||
from agent_client import ask_agent, get_agent_status
|
||||
from hotkey_listener import HotkeyListener, KeyCapturer, str_to_display
|
||||
|
||||
ctk.set_appearance_mode("dark")
|
||||
ctk.set_default_color_theme("blue")
|
||||
|
||||
# ── UI-Event-Queue (thread-safe) ─────────────────────────────────────────
|
||||
_ui_queue: queue.Queue = queue.Queue()
|
||||
|
||||
def ui_event(event: str, data=None):
|
||||
_ui_queue.put((event, data))
|
||||
|
||||
|
||||
# ── Hauptfenster ─────────────────────────────────────────────────────────
|
||||
class HermesApp(ctk.CTk):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.settings = cfg.load()
|
||||
self._hotkey: HotkeyListener | None = None
|
||||
self._worker: threading.Thread | None = None
|
||||
|
||||
self.title("Hermes Voice")
|
||||
self.geometry("400x520")
|
||||
self.resizable(False, False)
|
||||
self.protocol("WM_DELETE_WINDOW", self._on_close)
|
||||
|
||||
self._build_ui()
|
||||
self._start_hotkey()
|
||||
self.after(100, self._poll_queue)
|
||||
|
||||
# ── Layout ───────────────────────────────────────────────────────────
|
||||
def _build_ui(self):
|
||||
self.grid_columnconfigure(0, weight=1)
|
||||
self.grid_rowconfigure(2, weight=1)
|
||||
|
||||
# Header
|
||||
header = ctk.CTkFrame(self, height=50, corner_radius=0)
|
||||
header.grid(row=0, column=0, sticky="ew")
|
||||
header.grid_columnconfigure(0, weight=1)
|
||||
ctk.CTkLabel(header, text="⚡ Hermes Voice",
|
||||
font=ctk.CTkFont(size=16, weight="bold")).grid(
|
||||
row=0, column=0, padx=16, pady=12, sticky="w")
|
||||
ctk.CTkButton(header, text="⚙", width=36, height=36,
|
||||
command=self._open_settings).grid(
|
||||
row=0, column=1, padx=8, pady=8)
|
||||
|
||||
# Status
|
||||
status_frame = ctk.CTkFrame(self, corner_radius=12)
|
||||
status_frame.grid(row=1, column=0, padx=16, pady=(16, 8), sticky="ew")
|
||||
status_frame.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self._status_dot = ctk.CTkLabel(status_frame, text="●",
|
||||
font=ctk.CTkFont(size=32),
|
||||
text_color="#22c55e")
|
||||
self._status_dot.grid(row=0, column=0, pady=(12, 4))
|
||||
|
||||
self._status_label = ctk.CTkLabel(status_frame, text="Bereit",
|
||||
font=ctk.CTkFont(size=14))
|
||||
self._status_label.grid(row=1, column=0, pady=(0, 4))
|
||||
|
||||
self._hotkey_label = ctk.CTkLabel(
|
||||
status_frame,
|
||||
text=f"[ {str_to_display(self.settings['hotkey'])} ] gedrückt halten",
|
||||
font=ctk.CTkFont(size=11),
|
||||
text_color="gray",
|
||||
)
|
||||
self._hotkey_label.grid(row=2, column=0, pady=(0, 12))
|
||||
|
||||
# Conversation log
|
||||
log_frame = ctk.CTkFrame(self, corner_radius=12)
|
||||
log_frame.grid(row=2, column=0, padx=16, pady=8, sticky="nsew")
|
||||
log_frame.grid_columnconfigure(0, weight=1)
|
||||
log_frame.grid_rowconfigure(1, weight=1)
|
||||
|
||||
ctk.CTkLabel(log_frame, text="Letzte Konversation",
|
||||
font=ctk.CTkFont(size=11), text_color="gray").grid(
|
||||
row=0, column=0, padx=12, pady=(8, 0), sticky="w")
|
||||
|
||||
self._log = ctk.CTkTextbox(log_frame, font=ctk.CTkFont(size=12),
|
||||
state="disabled", wrap="word")
|
||||
self._log.grid(row=1, column=0, padx=8, pady=(4, 8), sticky="nsew")
|
||||
|
||||
# Footer
|
||||
footer = ctk.CTkFrame(self, height=40, corner_radius=0)
|
||||
footer.grid(row=3, column=0, sticky="ew")
|
||||
footer.grid_columnconfigure(0, weight=1)
|
||||
|
||||
self._screenshot_var = ctk.BooleanVar(
|
||||
value=self.settings["screenshot_enabled"])
|
||||
ctk.CTkCheckBox(footer, text="Screenshot mitsenden",
|
||||
variable=self._screenshot_var,
|
||||
command=self._toggle_screenshot).grid(
|
||||
row=0, column=0, padx=16, pady=8, sticky="w")
|
||||
|
||||
ctk.CTkButton(footer, text="Stop", width=60, height=28,
|
||||
fg_color="#dc2626", hover_color="#b91c1c",
|
||||
command=stop_speaking).grid(
|
||||
row=0, column=1, padx=8, pady=8)
|
||||
|
||||
# ── Hotkey ───────────────────────────────────────────────────────────
|
||||
def _start_hotkey(self):
|
||||
if self._hotkey:
|
||||
self._hotkey.stop()
|
||||
self._hotkey = HotkeyListener(
|
||||
self.settings["hotkey"],
|
||||
on_press_cb=self._hotkey_pressed,
|
||||
on_release_cb=self._hotkey_released,
|
||||
)
|
||||
self._hotkey.start()
|
||||
|
||||
def _hotkey_pressed(self):
|
||||
ui_event("status", ("recording", "🔴 Ich höre…", "#ef4444"))
|
||||
play_ding()
|
||||
start_recording()
|
||||
|
||||
def _hotkey_released(self):
|
||||
audio = stop_recording()
|
||||
ui_event("status", ("thinking", "💭 Denke…", "#3b82f6"))
|
||||
self._worker = threading.Thread(
|
||||
target=self._process, args=(audio,), daemon=True)
|
||||
self._worker.start()
|
||||
|
||||
def _process(self, audio):
|
||||
text = transcribe(audio,
|
||||
self.settings["whisper_model"],
|
||||
self.settings["language"])
|
||||
if not text:
|
||||
ui_event("status", ("idle", "Bereit", "#22c55e"))
|
||||
return
|
||||
|
||||
ui_event("log_user", text)
|
||||
screenshot = (capture_screen()
|
||||
if self.settings["screenshot_enabled"] else None)
|
||||
|
||||
ui_event("status", ("thinking", "💭 Denke…", "#3b82f6"))
|
||||
response, tool_calls = ask_agent(text, screenshot, self.settings)
|
||||
|
||||
for tc in tool_calls:
|
||||
ui_event("log_tool", tc)
|
||||
|
||||
ui_event("log_hermes", response)
|
||||
ui_event("status", ("speaking", "🔊 Spricht…", "#a855f7"))
|
||||
speak(response, self.settings["tts_voice"], self.settings["tts_rate"])
|
||||
ui_event("status", ("idle", "Bereit", "#22c55e"))
|
||||
|
||||
# ── Queue-Polling ────────────────────────────────────────────────────
|
||||
def _poll_queue(self):
|
||||
while not _ui_queue.empty():
|
||||
event, data = _ui_queue.get_nowait()
|
||||
if event == "status":
|
||||
_, label, color = data
|
||||
self._status_label.configure(text=label)
|
||||
self._status_dot.configure(text_color=color)
|
||||
elif event == "log_user":
|
||||
self._append_log(f"Du: {data}")
|
||||
elif event == "log_hermes":
|
||||
self._append_log(f"Hermes: {data}\n")
|
||||
self.after(80, self._poll_queue)
|
||||
|
||||
def _append_log(self, text: str):
|
||||
self._log.configure(state="normal")
|
||||
self._log.insert("end", text + "\n")
|
||||
self._log.see("end")
|
||||
self._log.configure(state="disabled")
|
||||
|
||||
# ── Settings Dialog ──────────────────────────────────────────────────
|
||||
def _open_settings(self):
|
||||
SettingsWindow(self)
|
||||
|
||||
def _toggle_screenshot(self):
|
||||
self.settings["screenshot_enabled"] = self._screenshot_var.get()
|
||||
cfg.save(self.settings)
|
||||
|
||||
def apply_settings(self, new_settings: dict):
|
||||
self.settings = new_settings
|
||||
cfg.save(new_settings)
|
||||
self._hotkey.update_key(new_settings["hotkey"])
|
||||
self._hotkey_label.configure(
|
||||
text=f"[ {str_to_display(new_settings['hotkey'])} ] gedrückt halten")
|
||||
self._screenshot_var.set(new_settings["screenshot_enabled"])
|
||||
|
||||
def _on_close(self):
|
||||
if self._hotkey:
|
||||
self._hotkey.stop()
|
||||
stop_speaking()
|
||||
self.destroy()
|
||||
|
||||
|
||||
# ── Settings Window ───────────────────────────────────────────────────────
|
||||
class SettingsWindow(ctk.CTkToplevel):
|
||||
def __init__(self, parent: HermesApp):
|
||||
super().__init__(parent)
|
||||
self._parent = parent
|
||||
self.title("Einstellungen")
|
||||
self.geometry("420x520")
|
||||
self.resizable(False, False)
|
||||
self.grab_set()
|
||||
self._s = dict(parent.settings)
|
||||
self._capturing = False
|
||||
self._build()
|
||||
|
||||
def _build(self):
|
||||
self.grid_columnconfigure(1, weight=1)
|
||||
row = 0
|
||||
|
||||
def label(text):
|
||||
nonlocal row
|
||||
ctk.CTkLabel(self, text=text, anchor="w").grid(
|
||||
row=row, column=0, padx=16, pady=6, sticky="w")
|
||||
|
||||
def entry(key, width=240):
|
||||
nonlocal row
|
||||
var = ctk.StringVar(value=str(self._s.get(key, "")))
|
||||
e = ctk.CTkEntry(self, textvariable=var, width=width)
|
||||
e.grid(row=row, column=1, padx=16, pady=6, sticky="ew")
|
||||
row += 1
|
||||
return var
|
||||
|
||||
def dropdown(key, options, width=200):
|
||||
nonlocal row
|
||||
var = ctk.StringVar(value=str(self._s.get(key, options[0])))
|
||||
dd = ctk.CTkOptionMenu(self, values=options, variable=var, width=width)
|
||||
dd.grid(row=row, column=1, padx=16, pady=6, sticky="w")
|
||||
row += 1
|
||||
return var
|
||||
|
||||
def toggle(key):
|
||||
nonlocal row
|
||||
var = ctk.BooleanVar(value=bool(self._s.get(key, False)))
|
||||
sw = ctk.CTkSwitch(self, text="", variable=var)
|
||||
sw.grid(row=row, column=1, padx=16, pady=6, sticky="w")
|
||||
row += 1
|
||||
return var
|
||||
|
||||
label("MC2 URL")
|
||||
self._url = entry("mc2_url")
|
||||
|
||||
label("Hotkey")
|
||||
self._hotkey_frame = ctk.CTkFrame(self, fg_color="transparent")
|
||||
self._hotkey_frame.grid(row=row, column=1, padx=16, pady=6, sticky="w")
|
||||
self._hotkey_display = ctk.CTkLabel(
|
||||
self._hotkey_frame,
|
||||
text=str_to_display(self._s.get("hotkey", "ctrl_r")),
|
||||
width=100,
|
||||
)
|
||||
self._hotkey_display.grid(row=0, column=0, padx=(0, 8))
|
||||
ctk.CTkButton(self._hotkey_frame, text="Aufnehmen", width=90,
|
||||
command=self._capture_hotkey).grid(row=0, column=1)
|
||||
row += 1
|
||||
|
||||
label("TTS Stimme")
|
||||
self._voice = dropdown("tts_voice", [
|
||||
"de-DE-SeraphinaMultilingualNeural",
|
||||
"de-DE-AmalaNeural",
|
||||
"de-DE-KatjaNeural",
|
||||
"de-DE-KillianNeural",
|
||||
"de-DE-ConradNeural",
|
||||
"en-US-AriaNeural",
|
||||
"en-US-JennyNeural",
|
||||
])
|
||||
|
||||
label("Whisper Modell")
|
||||
self._whisper = dropdown("whisper_model",
|
||||
["tiny", "base", "small", "medium"])
|
||||
|
||||
label("Sprache")
|
||||
self._lang = dropdown("language",
|
||||
["de", "en", "auto"])
|
||||
|
||||
label("Screenshot")
|
||||
self._screenshot = toggle("screenshot_enabled")
|
||||
|
||||
label("Monitor")
|
||||
self._monitor = dropdown("screenshot_monitor", ["1", "2", "3"])
|
||||
|
||||
label("Max. Tokens")
|
||||
self._tokens = entry("max_response_tokens", width=100)
|
||||
|
||||
row += 1
|
||||
ctk.CTkButton(self, text="Speichern", command=self._save).grid(
|
||||
row=row, column=0, columnspan=2, pady=16, padx=16, sticky="ew")
|
||||
|
||||
def _capture_hotkey(self):
|
||||
self._hotkey_display.configure(text="Taste drücken…")
|
||||
|
||||
def on_key(key_str):
|
||||
self._s["hotkey"] = key_str
|
||||
self.after(0, lambda: self._hotkey_display.configure(
|
||||
text=str_to_display(key_str)))
|
||||
|
||||
KeyCapturer(on_key).start()
|
||||
|
||||
def _save(self):
|
||||
self._s["mc2_url"] = self._url.get().strip()
|
||||
self._s["tts_voice"] = self._voice.get()
|
||||
self._s["whisper_model"] = self._whisper.get()
|
||||
self._s["language"] = self._lang.get()
|
||||
self._s["screenshot_enabled"] = self._screenshot.get()
|
||||
self._s["screenshot_monitor"] = int(self._monitor.get())
|
||||
try:
|
||||
self._s["max_response_tokens"] = int(self._tokens.get())
|
||||
except ValueError:
|
||||
pass
|
||||
self._parent.apply_settings(self._s)
|
||||
self.destroy()
|
||||
|
||||
|
||||
# ── Entry Point ───────────────────────────────────────────────────────────
|
||||
if __name__ == "__main__":
|
||||
app = HermesApp()
|
||||
app.mainloop()
|
||||
@@ -1,29 +0,0 @@
|
||||
# Hermes Voice Client — Dependencies
|
||||
# Installation: pip install -r requirements.txt
|
||||
|
||||
# Speech-to-Text (lokal auf CPU)
|
||||
faster-whisper>=1.0.0
|
||||
|
||||
# Audio I/O
|
||||
sounddevice>=0.4.6
|
||||
numpy>=1.24.0
|
||||
scipy>=1.11.0
|
||||
|
||||
# Text-to-Speech
|
||||
edge-tts>=6.1.9
|
||||
|
||||
# Screen Capture
|
||||
mss>=9.0.1
|
||||
Pillow>=10.0.0
|
||||
|
||||
# HTTP Client
|
||||
httpx>=0.27.0
|
||||
|
||||
# Audio Playback
|
||||
pygame>=2.5.0
|
||||
|
||||
# GUI
|
||||
customtkinter>=5.2.0
|
||||
|
||||
# Globaler Hotkey
|
||||
pynput>=1.7.6
|
||||
@@ -1,19 +0,0 @@
|
||||
import base64
|
||||
import io
|
||||
import mss
|
||||
from PIL import Image
|
||||
from config import SCREENSHOT_MONITOR, SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT
|
||||
|
||||
|
||||
def capture_screen() -> str:
|
||||
"""Screenshot des primären Monitors als base64-JPEG."""
|
||||
with mss.mss() as sct:
|
||||
monitor = sct.monitors[SCREENSHOT_MONITOR]
|
||||
raw = sct.grab(monitor)
|
||||
img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
|
||||
|
||||
img = img.resize((SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT), Image.LANCZOS)
|
||||
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=80)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
@@ -1,44 +0,0 @@
|
||||
@echo off
|
||||
:: Immer aus dem eigenen Verzeichnis laufen
|
||||
cd /d "%~dp0"
|
||||
|
||||
echo ========================================
|
||||
echo Hermes Voice Client -- Setup
|
||||
echo ========================================
|
||||
echo.
|
||||
|
||||
:: Python pruefen
|
||||
python --version >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo [FEHLER] Python nicht gefunden. Bitte Python 3.11+ installieren.
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
:: Venv anlegen falls nicht vorhanden
|
||||
if not exist ".venv" (
|
||||
echo [1/3] Erstelle virtuelle Umgebung...
|
||||
python -m venv .venv
|
||||
)
|
||||
|
||||
:: Dependencies installieren
|
||||
echo [2/3] Installiere Dependencies...
|
||||
.venv\Scripts\pip install --upgrade pip -q
|
||||
.venv\Scripts\pip install -r requirements.txt
|
||||
|
||||
:: Kurz-Check
|
||||
echo [3/3] Pruefe Konfiguration...
|
||||
.venv\Scripts\python -c "import customtkinter; import pynput; print('GUI + Hotkey OK')"
|
||||
|
||||
echo.
|
||||
echo ========================================
|
||||
echo Setup abgeschlossen!
|
||||
echo.
|
||||
echo Naechste Schritte:
|
||||
echo 1. start.bat ausfuehren
|
||||
echo 2. Einstellungen (Zahnrad) pruefen:
|
||||
echo - MC2 URL (Standard: 192.168.178.151:9001)
|
||||
echo - Hotkey (Standard: Rechte Strg)
|
||||
echo Optional: build.bat fuer .exe-Kompilierung
|
||||
echo ========================================
|
||||
pause
|
||||
@@ -1,4 +0,0 @@
|
||||
@echo off
|
||||
:: Hermes Voice Client starten (kein Konsolenfenster im Hintergrund)
|
||||
cd /d "%~dp0"
|
||||
start "" .venv\Scripts\pythonw.exe main.py
|
||||
@@ -1,7 +0,0 @@
|
||||
node_modules/
|
||||
out/
|
||||
dist/
|
||||
*.log
|
||||
# Avatar-VRM ist gross + Lizenz/Redistribution -> nicht committen (lokal/Box vorhalten)
|
||||
src/renderer/public/avatar.vrm
|
||||
src/renderer/public/vad/
|
||||
@@ -1,8 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "%~dp0"
|
||||
curl -s -m 5 http://127.0.0.1:8130/health > lucy_check.log 2>&1
|
||||
echo.>> lucy_check.log
|
||||
echo --- electron/vite/pocket laeuft? --->> lucy_check.log
|
||||
powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'pocket_server:app|electron-vite|lucy-desktop' } | ForEach-Object { $_.Name + ' ' + $_.ProcessId }" >> lucy_check.log 2>&1
|
||||
type lucy_check.log
|
||||
@@ -1,13 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-desktop"
|
||||
echo Beende haengende Versuche (ohne mich selbst)...
|
||||
powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.Name -eq 'electron.exe' -or $_.CommandLine -match 'electron-vite|pocket_server:app|start-lucy' } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -EA SilentlyContinue }"
|
||||
timeout /t 2 >nul
|
||||
echo Starte 'npm run dev' mit Log-Capture...
|
||||
start /b "" cmd /c "npm run dev > dev.log 2>&1"
|
||||
timeout /t 22 >nul
|
||||
echo === dev.log ===
|
||||
type dev.log
|
||||
echo.
|
||||
echo (laeuft im Hintergrund weiter, falls erfolgreich)
|
||||
@@ -1,12 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
REM Hygiene: Recon-Log mit Token entfernen
|
||||
del /q "F:\Coding Stuff\mission-control-2\box_recon.log" 2>nul
|
||||
echo Beende laufende Lucy-Instanz (pocket_server :8130 + Electron/Vite)...
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -Command "Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'pocket_server:app|lucy-desktop|electron-vite' } | ForEach-Object { Write-Output ('kill ' + $_.ProcessId + ' ' + $_.Name); Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }"
|
||||
timeout /t 2 >nul
|
||||
echo.
|
||||
echo Starte Lucy neu (frischer Voice-Server + Client mit erweitertem Umlaut-Fix)...
|
||||
start "" "F:\Coding Stuff\mission-control-2\client\lucy-desktop\start-lucy.cmd"
|
||||
timeout /t 3 >nul
|
||||
echo Fertig.
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -Command "$ws=New-Object -ComObject WScript.Shell; $p=Join-Path ([Environment]::GetFolderPath('Desktop')) 'Lucy.lnk'; $l=$ws.CreateShortcut($p); $l.TargetPath='F:\Coding Stuff\mission-control-2\client\lucy-desktop\start-lucy.cmd'; $l.WorkingDirectory='F:\Coding Stuff\mission-control-2\client\lucy-desktop'; $l.IconLocation='F:\Coding Stuff\mission-control-2\client\lucy-desktop\lucy.ico'; $l.Description='Lucy - lokaler Sprach-Companion starten'; $l.Save(); Write-Output ('OK: ' + $p)" > lucy_shortcut.log 2>&1
|
||||
type lucy_shortcut.log
|
||||
echo.
|
||||
echo Fertig - "Lucy" liegt jetzt auf dem Desktop.
|
||||
pause
|
||||
@@ -1,20 +0,0 @@
|
||||
import { defineConfig } from "electron-vite"
|
||||
import react from "@vitejs/plugin-react"
|
||||
import { resolve } from "path"
|
||||
|
||||
export default defineConfig({
|
||||
main: {
|
||||
build: { rollupOptions: { input: { index: resolve(__dirname, "src/main/index.ts") } } },
|
||||
},
|
||||
preload: {
|
||||
build: { rollupOptions: { input: { index: resolve(__dirname, "src/preload/index.ts") } } },
|
||||
},
|
||||
renderer: {
|
||||
root: resolve(__dirname, "src/renderer"),
|
||||
build: { rollupOptions: { input: { index: resolve(__dirname, "src/renderer/index.html") } } },
|
||||
// Silero-VAD-Assets (Worklet + ONNX + onnxruntime-WASM) liegen self-hosted in
|
||||
// src/renderer/public/vad/ (scripts/copy-vad-assets.mjs, läuft via npm postinstall) —
|
||||
// Pfade = baseAssetPath/onnxWASMBasePath in useVAD.ts.
|
||||
plugins: [react()],
|
||||
},
|
||||
})
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 16 KiB |
Generated
-4702
File diff suppressed because it is too large
Load Diff
@@ -1,39 +0,0 @@
|
||||
{
|
||||
"name": "lucy-desktop",
|
||||
"version": "0.1.0",
|
||||
"description": "Lucy — lokaler Sprach-Companion (Hermes-Hirn auf der Box, lokale Stimme auf der 9070 XT)",
|
||||
"main": "out/main/index.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "electron-vite dev",
|
||||
"build": "electron-vite build",
|
||||
"preview": "electron-vite preview",
|
||||
"postinstall": "node scripts/copy-vad-assets.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pixiv/three-vrm": "^3.4.0",
|
||||
"@pixiv/three-vrm-animation": "^3.5.4",
|
||||
"@react-three/drei": "^9.114.0",
|
||||
"@react-three/fiber": "^8.17.10",
|
||||
"@ricky0123/vad-web": "^0.0.30",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-markdown": "^9.1.0",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"three": "^0.169.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.3.12",
|
||||
"@types/react-dom": "^18.3.1",
|
||||
"@types/three": "^0.169.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"electron": "^43.0.0",
|
||||
"electron-vite": "^3.0.0",
|
||||
"typescript": "^5.6.3",
|
||||
"vite": "^6.0.3"
|
||||
},
|
||||
"allowScripts": {
|
||||
"electron": true,
|
||||
"esbuild": true
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
// Kopiert die Silero-VAD-Assets (Worklet + ONNX-Modell + onnxruntime-WASM) nach public/vad/,
|
||||
// damit die App sie self-hosted lädt (offlinefähig, kein CDN). Läuft via npm postinstall.
|
||||
// public/ wird von Vite in Dev UND Build verbatim ausgeliefert -> ein Pfad für beide Welten.
|
||||
import { copyFileSync, mkdirSync, readdirSync } from "node:fs"
|
||||
import { dirname, join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..")
|
||||
const dest = join(root, "src", "renderer", "public", "vad")
|
||||
mkdirSync(dest, { recursive: true })
|
||||
|
||||
const jobs = [
|
||||
[join(root, "node_modules", "@ricky0123", "vad-web", "dist"), (f) => f === "vad.worklet.bundle.min.js" || f.endsWith(".onnx")],
|
||||
[join(root, "node_modules", "onnxruntime-web", "dist"), (f) => f.endsWith(".wasm") || f.endsWith(".mjs")],
|
||||
]
|
||||
|
||||
let n = 0
|
||||
for (const [src, match] of jobs) {
|
||||
for (const f of readdirSync(src)) {
|
||||
if (match(f)) { copyFileSync(join(src, f), join(dest, f)); n++ }
|
||||
}
|
||||
}
|
||||
console.log(`[vad-assets] ${n} Dateien nach ${dest} kopiert.`)
|
||||
@@ -1,249 +0,0 @@
|
||||
import { app, BrowserWindow, ipcMain, Tray, Menu, globalShortcut, nativeImage, desktopCapturer, shell, screen, session } from "electron"
|
||||
import { spawn, ChildProcess } from "child_process"
|
||||
import { join } from "path"
|
||||
|
||||
// Lucy — Main-Prozess. Spawnt die lokale Stimme (pocket-tts, :8130) + verwaltet Fenster/Tray/Hotkey.
|
||||
// Fenster ist frameless+transparent: eigene Titelleiste im Renderer. Zwei Modi: 'full' (Arbeitsfenster)
|
||||
// und 'overlay' (kleine schwebende Lucy, always-on-top). pocket-tts ist CPU-kühl -> kein /wake-/sleep.
|
||||
|
||||
const PORT = process.env.LUCY_TTS_PORT || "8130"
|
||||
const TTS = `http://127.0.0.1:${PORT}`
|
||||
const TTS_DIR = process.env.LUCY_TTS_DIR || join(app.getAppPath(), "..", "lucy-tts")
|
||||
const PY = process.env.LUCY_TTS_PY || join(TTS_DIR, "ptts-venv", "Scripts", "python.exe")
|
||||
const HOTKEY = process.env.LUCY_HOTKEY || "CommandOrControl+Shift+Space"
|
||||
|
||||
const FULL_SIZE = { width: 1100, height: 720 }
|
||||
const OVERLAY_SIZE = { width: 360, height: 480 }
|
||||
|
||||
let win: BrowserWindow | null = null
|
||||
let tray: Tray | null = null
|
||||
let ttsProc: ChildProcess | null = null
|
||||
let mode: "full" | "overlay" = "full"
|
||||
let pinned = false
|
||||
let cursorTimer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
// MateEngine-Idee: Lucy folgt dem Mauszeiger. Der Zeiger ist auch AUSSERHALB des Fensters (Geistmodus),
|
||||
// daher global pollen (screen.getCursorScreenPoint) und die Richtung relativ zu Lucys Kopf an den Renderer geben.
|
||||
function startCursorTracking() {
|
||||
if (cursorTimer) return
|
||||
cursorTimer = setInterval(() => {
|
||||
if (!win || win.isDestroyed() || !win.isVisible()) return
|
||||
const b = win.getBounds()
|
||||
const p = screen.getCursorScreenPoint()
|
||||
const cx = b.x + b.width / 2
|
||||
const cy = b.y + b.height * 0.38 // ungefaehre Kopfhoehe im Fenster
|
||||
win.webContents.send("lucy:cursor", { x: (p.x - cx) / Math.max(1, b.width), y: (p.y - cy) / Math.max(1, b.height) })
|
||||
// Auto-Ducken: im Overlay nach Nicht-Naehe an den Rand; bei Annaeherung an den Rest hervorholen
|
||||
if (mode === "overlay" && !pinned && !boundsAnim) {
|
||||
const now = Date.now()
|
||||
const inWin = p.x >= b.x && p.y >= b.y && p.x <= b.x + b.width && p.y <= b.y + b.height
|
||||
if (inWin) lastNear = now
|
||||
if (!tucked && now - lastNear > AUTO_TUCK_MS) tuckToEdge()
|
||||
else if (tucked) {
|
||||
const wa = screen.getDisplayNearestPoint({ x: b.x, y: b.y }).workArea
|
||||
if (p.x >= wa.x + wa.width - SLIVER - 12 && p.y >= b.y - 12 && p.y <= b.y + b.height + 12) { lastNear = now; untuck() }
|
||||
}
|
||||
}
|
||||
}, 40) // ~25 Hz reicht fuers Blickfolgen, kostet kaum Last
|
||||
}
|
||||
function stopCursorTracking() { if (cursorTimer) { clearInterval(cursorTimer); cursorTimer = null } }
|
||||
|
||||
// --- An den Rand ducken (MateEngine-Idee) ---
|
||||
type Rect = { x: number; y: number; width: number; height: number }
|
||||
let tucked = false
|
||||
let prevBounds: Rect | null = null
|
||||
let lastNear = Date.now()
|
||||
let boundsAnim: ReturnType<typeof setInterval> | null = null
|
||||
const SLIVER = 58 // sichtbarer Rest am Rand, wenn geduckt
|
||||
const AUTO_TUCK_MS = 15000 // nach so langer Nicht-Naehe im Overlay automatisch ducken
|
||||
|
||||
function animateBounds(target: Rect, after?: () => void) {
|
||||
if (!win) return
|
||||
if (boundsAnim) { clearInterval(boundsAnim); boundsAnim = null }
|
||||
const start = win.getBounds(); const t0 = Date.now(); const dur = 260
|
||||
boundsAnim = setInterval(() => {
|
||||
if (!win || win.isDestroyed()) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; return }
|
||||
const k = Math.min(1, (Date.now() - t0) / dur)
|
||||
const e = k < 0.5 ? 2 * k * k : 1 - Math.pow(-2 * k + 2, 2) / 2 // easeInOutQuad
|
||||
win.setBounds({
|
||||
x: Math.round(start.x + (target.x - start.x) * e),
|
||||
y: Math.round(start.y + (target.y - start.y) * e),
|
||||
width: target.width, height: target.height,
|
||||
})
|
||||
if (k >= 1) { if (boundsAnim) clearInterval(boundsAnim); boundsAnim = null; after?.() }
|
||||
}, 16)
|
||||
}
|
||||
function tuckToEdge() {
|
||||
if (!win || mode !== "overlay" || tucked) return
|
||||
prevBounds = win.getBounds()
|
||||
const wa = screen.getDisplayNearestPoint({ x: prevBounds.x, y: prevBounds.y }).workArea
|
||||
animateBounds({ x: wa.x + wa.width - SLIVER, y: prevBounds.y, width: prevBounds.width, height: prevBounds.height })
|
||||
tucked = true; setTrayMenu()
|
||||
}
|
||||
function untuck() {
|
||||
if (!win || !tucked) return
|
||||
const t = prevBounds || win.getBounds()
|
||||
animateBounds({ x: t.x, y: t.y, width: t.width, height: t.height })
|
||||
tucked = false; setTrayMenu()
|
||||
}
|
||||
function toggleTuck() { if (tucked) untuck(); else tuckToEdge() }
|
||||
|
||||
async function ensureTtsServer() {
|
||||
try {
|
||||
const r = await fetch(`${TTS}/health`)
|
||||
if (r.ok) { console.log("[lucy] TTS-Server läuft bereits auf", TTS); return }
|
||||
} catch { /* selbst starten */ }
|
||||
console.log("[lucy] starte pocket_server:", PY, "(cwd:", TTS_DIR + ")")
|
||||
ttsProc = spawn(PY, ["-m", "uvicorn", "pocket_server:app", "--host", "127.0.0.1", "--port", PORT],
|
||||
{ cwd: TTS_DIR, windowsHide: true, env: {
|
||||
...process.env,
|
||||
// Bench-Optimum aus Phase B2 (docs/LUCY_TTS_PLAN.md): 4 Worker x 2 Threads = RTF 0,44
|
||||
// statt 0,67 seriell. Ohne diese Envs startete der Server SERIELL (workers=0) ->
|
||||
// TTS-TTFB 1,7-2,3 s je Antwort (Realtest 02.07.). Per Umgebung überschreibbar.
|
||||
LUCY_WORKERS: process.env.LUCY_WORKERS || "4",
|
||||
LUCY_WORKER_THREADS: process.env.LUCY_WORKER_THREADS || "2",
|
||||
} })
|
||||
ttsProc.stdout?.on("data", (d) => console.log("[tts]", d.toString().trimEnd()))
|
||||
ttsProc.stderr?.on("data", (d) => console.log("[tts]", d.toString().trimEnd()))
|
||||
ttsProc.on("error", (e) => console.error("[lucy] pocket_server-Spawn fehlgeschlagen:", e))
|
||||
ttsProc.on("exit", (code) => { console.log("[lucy] pocket_server beendet, code", code); ttsProc = null })
|
||||
}
|
||||
function stopTtsServer() {
|
||||
if (ttsProc && !ttsProc.killed) { try { ttsProc.kill() } catch { /* */ }; ttsProc = null }
|
||||
}
|
||||
|
||||
function applyMode(next: "full" | "overlay") {
|
||||
if (!win) return
|
||||
// beim Verlassen des Overlays ggf. Duck-Zustand zuruecksetzen (Position wiederherstellen)
|
||||
if (tucked && next !== "overlay") { tucked = false; if (prevBounds) win.setPosition(prevBounds.x, prevBounds.y) }
|
||||
mode = next
|
||||
const overlay = next === "overlay"
|
||||
const size = overlay ? OVERLAY_SIZE : FULL_SIZE
|
||||
win.setAlwaysOnTop(overlay || pinned, "floating")
|
||||
win.setSize(size.width, size.height, false)
|
||||
win.setResizable(!overlay)
|
||||
if (overlay) win.setMinimumSize(280, 360); else win.setMinimumSize(420, 420)
|
||||
win.webContents.send("lucy:mode", next)
|
||||
lastNear = Date.now(); setTrayMenu()
|
||||
}
|
||||
|
||||
function createWindow() {
|
||||
win = new BrowserWindow({
|
||||
width: FULL_SIZE.width, height: FULL_SIZE.height, minWidth: 420, minHeight: 420,
|
||||
frame: false, transparent: true, backgroundColor: "#00000000", resizable: true,
|
||||
title: "Lucy", show: false,
|
||||
webPreferences: {
|
||||
preload: join(__dirname, "../preload/index.mjs"),
|
||||
sandbox: false, webSecurity: false,
|
||||
},
|
||||
})
|
||||
if (process.env.ELECTRON_RENDERER_URL) win.loadURL(process.env.ELECTRON_RENDERER_URL)
|
||||
else win.loadFile(join(__dirname, "../renderer/index.html"))
|
||||
win.once("ready-to-show", () => win?.show())
|
||||
win.on("closed", () => { win = null })
|
||||
}
|
||||
|
||||
function showWin() {
|
||||
if (!win) { createWindow(); return }
|
||||
if (win.isMinimized()) win.restore()
|
||||
if (tucked) untuck()
|
||||
win.show(); win.focus()
|
||||
}
|
||||
|
||||
function lucyIcon() {
|
||||
// 16x16 Icon zur Laufzeit (silberblauer Kreis) -> sichtbar im Tray, ohne Asset-Datei
|
||||
const s = 16, buf = Buffer.alloc(s * s * 4)
|
||||
for (let y = 0; y < s; y++) for (let x = 0; x < s; x++) {
|
||||
const i = (y * s + x) * 4
|
||||
const dx = x - 7.5, dy = y - 7.5, inside = dx * dx + dy * dy <= 56
|
||||
buf[i] = 0xfc; buf[i + 1] = 0xd3; buf[i + 2] = 0x7d; buf[i + 3] = inside ? 0xff : 0x00 // BGRA
|
||||
}
|
||||
return nativeImage.createFromBitmap(buf, { width: s, height: s })
|
||||
}
|
||||
|
||||
function setTrayMenu() {
|
||||
if (!tray) return
|
||||
tray.setContextMenu(Menu.buildFromTemplate([
|
||||
{ label: "Lucy zeigen", click: showWin },
|
||||
{ label: "Overlay-Modus", type: "checkbox", checked: mode === "overlay",
|
||||
click: (mi) => { applyMode(mi.checked ? "overlay" : "full"); showWin() } },
|
||||
{ label: tucked ? "Hervorholen" : "An den Rand ducken", enabled: mode === "overlay", click: toggleTuck },
|
||||
{ type: "separator" },
|
||||
{ label: "Beenden", click: () => { app.quit() } },
|
||||
]))
|
||||
}
|
||||
function buildTray() {
|
||||
tray = new Tray(lucyIcon())
|
||||
tray.setToolTip("Lucy")
|
||||
setTrayMenu()
|
||||
tray.on("click", showWin)
|
||||
}
|
||||
|
||||
function registerHotkey() {
|
||||
globalShortcut.unregisterAll()
|
||||
// Global: Sprach-Aufnahme an/aus (Toggle, da Global-Shortcuts kein key-up liefern)
|
||||
const ok = globalShortcut.register(HOTKEY, () => { showWin(); win?.webContents.send("lucy:hotkey") })
|
||||
if (!ok) console.error("[lucy] Global-Hotkey-Registrierung fehlgeschlagen:", HOTKEY)
|
||||
}
|
||||
|
||||
// ---- IPC vom Renderer (Titelleiste/Modus) ----
|
||||
ipcMain.handle("win:minimize", () => win?.minimize())
|
||||
ipcMain.handle("win:close", () => win?.hide()) // in den Tray, bleibt warm; echtes Beenden über Tray
|
||||
ipcMain.handle("win:setMode", (_e, m: "full" | "overlay") => { applyMode(m); return mode })
|
||||
ipcMain.handle("win:getMode", () => mode)
|
||||
ipcMain.handle("win:togglePin", () => { pinned = !pinned; win?.setAlwaysOnTop(pinned || mode === "overlay", "floating"); return pinned })
|
||||
|
||||
// Links im echten System-Browser oeffnen (nicht im App-Fenster); nur http(s) erlauben
|
||||
ipcMain.handle("open:external", (_e, url: string) => { if (/^https?:\/\//i.test(url)) void shell.openExternal(url) })
|
||||
|
||||
// Geistmodus (Overlay v2): Fenster durchklickbar machen -> Lucy "sitzt" auf dem Desktop, du arbeitest
|
||||
// dahinter weiter. forward:true liefert weiter Hover-Events an den Renderer, damit er ueber Lucys
|
||||
// Bedienelementen kurzzeitig wieder klickbar schalten kann.
|
||||
ipcMain.handle("win:setClickThrough", (_e, on: boolean) => {
|
||||
win?.setIgnoreMouseEvents(!!on, { forward: true }); return !!on
|
||||
})
|
||||
|
||||
// ---- Bildschirm-Sicht (Lucys Augen) ----
|
||||
const CAP = { width: 1280, height: 720 }
|
||||
ipcMain.handle("screen:capture", async () => {
|
||||
const sources = await desktopCapturer.getSources({ types: ["screen"], thumbnailSize: CAP })
|
||||
return sources.map((s) => s.thumbnail.toDataURL()) // ALLE Bildschirme (Multi-Monitor) als PNG-DataURLs
|
||||
})
|
||||
ipcMain.handle("screen:listWindows", async () => {
|
||||
const sources = await desktopCapturer.getSources({ types: ["window"], thumbnailSize: { width: 0, height: 0 } })
|
||||
return sources.filter((s) => s.name && s.name !== "Lucy").map((s) => ({ id: s.id, name: s.name }))
|
||||
})
|
||||
ipcMain.handle("screen:captureWindow", async (_e, id: string) => {
|
||||
const sources = await desktopCapturer.getSources({ types: ["window"], thumbnailSize: CAP })
|
||||
return sources.find((s) => s.id === id)?.thumbnail.toDataURL() ?? null
|
||||
})
|
||||
// Fuzzy-Fenstersuche: bester Treffer für einen im Gesagten erkannten App-/Fensternamen
|
||||
ipcMain.handle("screen:captureByName", async (_e, query: string) => {
|
||||
const q = query.toLowerCase().trim()
|
||||
const sources = await desktopCapturer.getSources({ types: ["window"], thumbnailSize: CAP })
|
||||
const cand = sources.filter((s) => s.name && s.name !== "Lucy")
|
||||
const hit = cand.find((s) => s.name.toLowerCase().includes(q))
|
||||
|| cand.find((s) => q.split(/\s+/).some((w) => w.length > 2 && s.name.toLowerCase().includes(w)))
|
||||
return hit ? { name: hit.name, image: hit.thumbnail.toDataURL() } : null
|
||||
})
|
||||
|
||||
app.whenReady().then(() => {
|
||||
void ensureTtsServer()
|
||||
createWindow()
|
||||
buildTray()
|
||||
registerHotkey()
|
||||
startCursorTracking()
|
||||
// Tanzen zur Musik: System-Audio (Loopback) fuer getDisplayMedia bereitstellen, ohne System-Picker
|
||||
session.defaultSession.setDisplayMediaRequestHandler((_req, callback) => {
|
||||
desktopCapturer.getSources({ types: ["screen"] })
|
||||
.then((sources) => callback({ video: sources[0], audio: "loopback" }))
|
||||
.catch(() => { try { callback({}) } catch { /* */ } })
|
||||
}, { useSystemPicker: false })
|
||||
app.on("activate", () => { if (BrowserWindow.getAllWindows().length === 0) createWindow() })
|
||||
})
|
||||
|
||||
app.on("window-all-closed", () => { /* im Tray weiterlaufen lassen; Beenden nur über Tray/Quit */ })
|
||||
app.on("before-quit", () => {
|
||||
if (boundsAnim) clearInterval(boundsAnim)
|
||||
stopCursorTracking(); stopTtsServer(); globalShortcut.unregisterAll(); tray?.destroy()
|
||||
})
|
||||
@@ -1,34 +0,0 @@
|
||||
import { contextBridge, ipcRenderer } from "electron"
|
||||
|
||||
// Schmale Bruecke Renderer -> Main: Fenster-Chrome (frameless), Modus-Umschaltung, Tray/Hotkey-Events.
|
||||
contextBridge.exposeInMainWorld("lucy", {
|
||||
minimize: () => ipcRenderer.invoke("win:minimize"),
|
||||
close: () => ipcRenderer.invoke("win:close"),
|
||||
setMode: (m: "full" | "overlay") => ipcRenderer.invoke("win:setMode", m),
|
||||
getMode: () => ipcRenderer.invoke("win:getMode"),
|
||||
togglePin: () => ipcRenderer.invoke("win:togglePin"),
|
||||
// Bildschirm-Sicht
|
||||
captureScreen: () => ipcRenderer.invoke("screen:capture"), // alle Monitore -> string[]
|
||||
listWindows: () => ipcRenderer.invoke("screen:listWindows"),
|
||||
captureWindow: (id: string) => ipcRenderer.invoke("screen:captureWindow", id),
|
||||
captureByName: (q: string) => ipcRenderer.invoke("screen:captureByName", q),
|
||||
openExternal: (url: string) => ipcRenderer.invoke("open:external", url), // Link im System-Browser
|
||||
setClickThrough: (on: boolean) => ipcRenderer.invoke("win:setClickThrough", on), // Geistmodus (Overlay v2)
|
||||
// Events aus dem Main-Prozess (Tray/Hotkey/Modus)
|
||||
onMode: (cb: (m: "full" | "overlay") => void) => {
|
||||
const h = (_e: unknown, m: "full" | "overlay") => cb(m)
|
||||
ipcRenderer.on("lucy:mode", h)
|
||||
return () => ipcRenderer.removeListener("lucy:mode", h)
|
||||
},
|
||||
onHotkey: (cb: () => void) => {
|
||||
const h = () => cb()
|
||||
ipcRenderer.on("lucy:hotkey", h)
|
||||
return () => ipcRenderer.removeListener("lucy:hotkey", h)
|
||||
},
|
||||
// Mauszeiger-Position (relativ zu Lucys Kopf) -> Blickfolgen
|
||||
onCursor: (cb: (p: { x: number; y: number }) => void) => {
|
||||
const h = (_e: unknown, p: { x: number; y: number }) => cb(p)
|
||||
ipcRenderer.on("lucy:cursor", h)
|
||||
return () => ipcRenderer.removeListener("lucy:cursor", h)
|
||||
},
|
||||
})
|
||||
@@ -1,13 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta http-equiv="Content-Security-Policy" content="default-src 'self' 'unsafe-inline' data: blob: http://127.0.0.1:8130 http://192.168.178.151:9001 ws://localhost:*; media-src 'self' blob: data: http://127.0.0.1:8130; img-src 'self' data: blob:; script-src 'self' 'unsafe-inline'" />
|
||||
<title>Lucy</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,11 +0,0 @@
|
||||
# VRMA-Animationen — Quelle & Lizenz
|
||||
|
||||
Die `.vrma`-Bewegungs-Clips in diesem Ordner stammen aus:
|
||||
|
||||
**tk256ailab/vrm-viewer** — https://github.com/tk256ailab/vrm-viewer
|
||||
Lizenz: **MIT** · Copyright (c) 2025 TK256
|
||||
|
||||
Die MIT-Lizenz erlaubt Nutzung, Modifikation und Weitergabe; der Copyright-
|
||||
Hinweis ist beizubehalten (= diese Datei).
|
||||
|
||||
Genutzte Clips: Relax, Thinking, LookAround, Surprised, Sad, Clapping, Goodbye, Blush.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,234 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { Avatar3D } from "./components/Avatar3D"
|
||||
import { AuraGlow } from "./components/AuraGlow"
|
||||
import { ChatMarkdown } from "./components/ChatMarkdown"
|
||||
import { useVoiceAgent } from "./lib/voice/useVoiceAgent"
|
||||
import { useDanceAudio } from "./lib/voice/useDanceAudio"
|
||||
|
||||
const FIXED_AVATAR = "/avatar.vrm"
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
warming: "Lucy waermt auf …",
|
||||
idle: "Bereit — halte zum Sprechen",
|
||||
listening: "Hoere zu …",
|
||||
transcribing: "Verstehe …",
|
||||
thinking: "Lucy denkt …",
|
||||
speaking: "Lucy spricht …",
|
||||
error: "Fehler",
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
const { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset,
|
||||
inputMode, setInputMode, vadListening, lookAtScreen } = useVoiceAgent()
|
||||
const { danceLevel, dancing, toggleDance } = useDanceAudio()
|
||||
const holding = useRef(false)
|
||||
const endRef = useRef<HTMLDivElement>(null)
|
||||
const cursor = useRef({ x: 0, y: 0 }) // Mauszeiger relativ zu Lucys Kopf (Blickfolgen)
|
||||
const pat = useRef(0) // Zaehler: Antippen/Streicheln des Avatars
|
||||
const [mode, setMode] = useState<"full" | "overlay">("full")
|
||||
const [pinned, setPinned] = useState(false)
|
||||
const [ghost, setGhost] = useState(false) // Geistmodus: Fenster durchklickbar (nur Overlay)
|
||||
|
||||
useEffect(() => { endRef.current?.scrollIntoView({ behavior: "smooth", block: "end" }) }, [messages])
|
||||
|
||||
// Fenster-Modus vom Main synchronisieren (Tray/Hotkey koennen ihn aendern)
|
||||
useEffect(() => {
|
||||
window.lucy?.getMode().then(setMode).catch(() => {})
|
||||
return window.lucy?.onMode(setMode)
|
||||
}, [])
|
||||
|
||||
// Geistmodus (Overlay v2): Fenster durchklickbar, AUSSER der Zeiger ist ueber Lucys Bedienelementen
|
||||
// (.clickable). So sitzt sie auf dem Desktop, du arbeitest dahinter weiter, kannst sie aber bedienen.
|
||||
useEffect(() => {
|
||||
if (!(ghost && mode === "overlay")) { void window.lucy?.setClickThrough(false); return }
|
||||
let ignoring = false
|
||||
const set = (ig: boolean) => { if (ig !== ignoring) { ignoring = ig; void window.lucy?.setClickThrough(ig) } }
|
||||
set(true)
|
||||
const onMove = (e: MouseEvent) => {
|
||||
const el = document.elementFromPoint(e.clientX, e.clientY) as HTMLElement | null
|
||||
set(!(el && el.closest(".clickable")))
|
||||
}
|
||||
window.addEventListener("mousemove", onMove)
|
||||
return () => { window.removeEventListener("mousemove", onMove); void window.lucy?.setClickThrough(false) }
|
||||
}, [ghost, mode])
|
||||
useEffect(() => { if (mode !== "overlay") setGhost(false) }, [mode]) // Geistmodus nur im Overlay
|
||||
|
||||
// Mauszeiger-Verfolgung (MateEngine-Idee): Position vom Main-Prozess in den Avatar-Blick fuettern
|
||||
useEffect(() => window.lucy?.onCursor((p) => { cursor.current = p }), [])
|
||||
|
||||
// Leertaste = Push-to-talk (halten)
|
||||
useEffect(() => {
|
||||
const isField = (el: EventTarget | null) =>
|
||||
el instanceof HTMLElement && /^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName)
|
||||
const down = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || e.repeat || holding.current || isField(e.target)) return
|
||||
e.preventDefault(); holding.current = true; pressStart()
|
||||
}
|
||||
const up = (e: KeyboardEvent) => {
|
||||
if (e.code !== "Space" || !holding.current) return
|
||||
e.preventDefault(); holding.current = false; pressEnd()
|
||||
}
|
||||
window.addEventListener("keydown", down); window.addEventListener("keyup", up)
|
||||
return () => { window.removeEventListener("keydown", down); window.removeEventListener("keyup", up) }
|
||||
}, [pressStart, pressEnd])
|
||||
|
||||
// Global-Hotkey = TOGGLE (Global-Shortcuts liefern kein key-up) -> Druck startet/stoppt Aufnahme
|
||||
const recRef = useRef(recording); recRef.current = recording
|
||||
useEffect(() => {
|
||||
return window.lucy?.onHotkey(() => {
|
||||
if (recRef.current || holding.current) { holding.current = false; pressEnd() }
|
||||
else { holding.current = true; pressStart() }
|
||||
})
|
||||
}, [pressStart, pressEnd])
|
||||
|
||||
const toggleMode = useCallback(() => {
|
||||
const next = mode === "full" ? "overlay" : "full"
|
||||
window.lucy?.setMode(next).then(setMode).catch(() => setMode(next))
|
||||
}, [mode])
|
||||
const togglePin = useCallback(() => { window.lucy?.togglePin().then(setPinned).catch(() => {}) }, [])
|
||||
|
||||
const speaking = status === "speaking"
|
||||
const busy = status === "transcribing" || status === "thinking"
|
||||
const statusColor = status === "error" ? "#f87171" : speaking ? "#7dd3fc" : "#9ca3af"
|
||||
const overlay = mode === "overlay"
|
||||
const lastAssistant = [...messages].reverse().find((m) => m.role === "assistant")?.text?.trim() || ""
|
||||
// Overlay-Blasen: Links/Code aus der letzten Antwort griffbereit (im Overlay gibt es kein Chat-Panel)
|
||||
const ovLinks = overlay ? Array.from(new Set(lastAssistant.match(/https?:\/\/[^\s)]+/g) || [])).slice(0, 3) : []
|
||||
const ovCode = overlay ? (lastAssistant.match(/```[\w]*\n?([\s\S]*?)```/)?.[1] || "").trim() : ""
|
||||
const shortUrl = (u: string) => u.replace(/^https?:\/\/(www\.)?/, "")
|
||||
|
||||
const micBtn = (
|
||||
<button
|
||||
className={"mic" + (recording ? " rec" : "") + (!ready ? " disabled" : "")}
|
||||
disabled={!ready}
|
||||
onPointerDown={(e) => { if (!ready) return; e.preventDefault(); holding.current = true; pressStart() }}
|
||||
onPointerUp={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
onPointerLeave={() => { if (holding.current) { holding.current = false; pressEnd() } }}
|
||||
title={ready ? "Gedrueckt halten zum Sprechen (oder Leertaste)" : "Lucy waermt noch auf …"}
|
||||
>
|
||||
<svg width="30" height="30" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3z" />
|
||||
<path d="M19 10v2a7 7 0 0 1-14 0v-2" /><line x1="12" y1="19" x2="12" y2="22" />
|
||||
</svg>
|
||||
</button>
|
||||
)
|
||||
|
||||
return (
|
||||
<div className={"app " + mode}>
|
||||
<div className="titlebar clickable">
|
||||
<div className="drag">
|
||||
<span className="dot" style={{ background: statusColor }} />
|
||||
<span className="ttl">Lucy</span>
|
||||
{!overlay && <span className="ttlStatus">{error || STATUS_LABEL[status]}</span>}
|
||||
</div>
|
||||
<div className="winbtns">
|
||||
<button className="wb" onClick={() => void lookAtScreen()} title="Lucy auf den Bildschirm schauen lassen" disabled={!ready}>👁</button>
|
||||
{overlay && <button className="wb" onClick={() => setGhost((g) => !g)} title="Geistmodus: durchklickbar" data-on={ghost}>👻</button>}
|
||||
<button className="wb" onClick={togglePin} title="Immer im Vordergrund" data-on={pinned}>📌</button>
|
||||
<button className="wb" onClick={toggleMode} title={overlay ? "Vollfenster" : "Overlay-Modus"}>{overlay ? "▣" : "▢"}</button>
|
||||
<button className="wb" onClick={() => window.lucy?.minimize()} title="Minimieren">—</button>
|
||||
<button className="wb close" onClick={() => window.lucy?.close()} title="In den Tray">✕</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body">
|
||||
<div className="stage">
|
||||
<div className="avatarWrap" onClick={() => { pat.current += 1 }}
|
||||
title={overlay ? "Ziehen zum Verschieben · Antippen zum Streicheln" : "Antippen zum Streicheln"}>
|
||||
<div className="digitalRoom"><div className="dust" /></div>
|
||||
<AuraGlow status={status} audioLevel={audioLevel} />
|
||||
<Avatar3D url={FIXED_AVATAR} audioLevel={audioLevel} emotion={emotion} status={status} cursor={cursor} pat={pat} dance={danceLevel} controls={!overlay} />
|
||||
{speaking && lastAssistant && <div className="subtitle">{lastAssistant}</div>}
|
||||
{!ready && (
|
||||
<div className="warmOverlay">
|
||||
<span className="spinner big" />
|
||||
<div className="warmTitle">Lucy waermt auf …</div>
|
||||
{!overlay && <div className="warmSub">Die lokale Stimme wird vorbereitet — einen Moment, Commander.</div>}
|
||||
</div>
|
||||
)}
|
||||
{overlay && (ovLinks.length > 0 || ovCode) && (
|
||||
<div className="overlayBubbles clickable">
|
||||
{ovLinks.map((l, i) => (
|
||||
<div className="bubbleChip" key={"l" + i}>
|
||||
<span className="chipTxt" title={l}>🔗 {shortUrl(l)}</span>
|
||||
<button onClick={() => void window.lucy?.openExternal(l)} title="Im Browser öffnen">↗</button>
|
||||
<button onClick={() => { void navigator.clipboard.writeText(l).catch(() => {}) }} title="Link kopieren">⧉</button>
|
||||
</div>
|
||||
))}
|
||||
{ovCode && (
|
||||
<div className="bubbleChip">
|
||||
<span className="chipTxt">⌘ Code-Schnipsel</span>
|
||||
<button onClick={() => { void navigator.clipboard.writeText(ovCode).catch(() => {}) }} title="Code kopieren">⧉</button>
|
||||
<button onClick={toggleMode} title="In der App ansehen">▣</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{overlay && (
|
||||
<div className="overlayCaption">
|
||||
{busy && <span className="spinner" />}
|
||||
<span>{error || STATUS_LABEL[status]}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="controls clickable">
|
||||
{!overlay && (
|
||||
<div className="status" style={{ color: statusColor }}>
|
||||
{busy && <span className="spinner" />}
|
||||
<span>{error || STATUS_LABEL[status]}</span>
|
||||
</div>
|
||||
)}
|
||||
{micBtn}
|
||||
{ready && (
|
||||
<div className="modeToggle" title="Eingabe-Modus">
|
||||
<button className={inputMode === "ptt" ? "on" : ""} onClick={() => setInputMode("ptt")}>Push-to-talk</button>
|
||||
<button className={inputMode === "vad" ? "on" : ""} onClick={() => setInputMode("vad")}>Freisprechen</button>
|
||||
</div>
|
||||
)}
|
||||
{ready && (
|
||||
<button className={"danceBtn" + (dancing ? " on" : "")} onClick={toggleDance}
|
||||
title="Zur Musik tanzen (nimmt den System-Ton ab)">{dancing ? "⏹ Tanz aus" : "💃 Tanzen"}</button>
|
||||
)}
|
||||
{!overlay && (
|
||||
<div className="hint">
|
||||
{inputMode === "vad"
|
||||
? <>Freisprechen aktiv{vadListening ? <span className="vadDot" /> : null} · <kbd>Leertaste</kbd> geht auch · Hotkey global</>
|
||||
: <>Halten zum Sprechen · <kbd>Leertaste</kbd> · Hotkey global</>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!overlay && (
|
||||
<div className="side">
|
||||
<div className="sideHead">
|
||||
<span>Gespräch</span>
|
||||
<button className="iconBtn" onClick={reset} title="Neues Gespräch">↺</button>
|
||||
</div>
|
||||
<div className="conv">
|
||||
{messages.length === 0 && (
|
||||
<p className="empty">Halte den Knopf (oder die Leertaste) und sprich. Lucy hört zu, denkt mit
|
||||
Hermes' vollem Gedächtnis und antwortet in ihrer lokalen Stimme.</p>
|
||||
)}
|
||||
{messages.map((m, i) => (
|
||||
<div key={i} className={"msg " + (m.role === "user" ? "user" : "assistant")}>
|
||||
<span className="who">{m.role === "user" ? "Du" : "Lucy"}</span>
|
||||
<div className="bubble">
|
||||
{m.text
|
||||
? (m.role === "assistant" ? <ChatMarkdown text={m.text} /> : m.text)
|
||||
: <span className="dots"><i /><i /><i /></span>}
|
||||
{m.role === "assistant" && m.text && (
|
||||
<button className="msgCopy" title="Antwort kopieren"
|
||||
onClick={() => { void navigator.clipboard.writeText(m.text).catch(() => {}) }}>⧉</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div ref={endRef} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { useEffect, useRef, type MutableRefObject } from "react"
|
||||
|
||||
// Status-als-Licht: reaktive Aura hinter Lucy. Farbe = Status (hört zu / denkt / spricht), Intensität
|
||||
// pulst mit dem Sprech-Pegel. Liest audioLevel per rAF (kein Re-Render pro Frame).
|
||||
type LevelRef = MutableRefObject<{ current: number }>
|
||||
|
||||
const COLORS: Record<string, [number, number, number]> = {
|
||||
warming: [120, 130, 150], idle: [125, 211, 252], listening: [56, 189, 248],
|
||||
transcribing: [167, 139, 250], thinking: [251, 191, 36], speaking: [52, 211, 153],
|
||||
error: [248, 113, 113],
|
||||
}
|
||||
|
||||
export function AuraGlow({ status, audioLevel }: { status: string; audioLevel: LevelRef }) {
|
||||
const ref = useRef<HTMLDivElement>(null)
|
||||
const statusRef = useRef(status); statusRef.current = status
|
||||
const phase = useRef(0)
|
||||
|
||||
useEffect(() => {
|
||||
let raf = 0, last = performance.now()
|
||||
const tick = (now: number) => {
|
||||
const dt = Math.min(0.05, (now - last) / 1000); last = now
|
||||
phase.current += dt
|
||||
const el = ref.current
|
||||
if (el) {
|
||||
const s = statusRef.current
|
||||
const [r, g, b] = COLORS[s] || COLORS.idle
|
||||
const lvl = audioLevel.current?.current ?? 0
|
||||
// Grund-Puls je Status + Sprech-Reaktivität
|
||||
const base = s === "thinking" || s === "transcribing" ? 0.45 + Math.sin(phase.current * 2.2) * 0.18
|
||||
: s === "listening" ? 0.55 + Math.sin(phase.current * 3) * 0.12
|
||||
: s === "speaking" ? 0.4 + Math.min(0.6, lvl * 1.8)
|
||||
: s === "error" ? 0.6
|
||||
: 0.32 + Math.sin(phase.current * 1.1) * 0.06 // idle: ruhiges Atmen
|
||||
const intensity = Math.max(0.15, Math.min(1, base))
|
||||
const scale = 1 + intensity * 0.18 + (s === "speaking" ? lvl * 0.25 : 0)
|
||||
el.style.background = `radial-gradient(circle at 50% 42%, rgba(${r},${g},${b},${0.5 * intensity}) 0%, rgba(${r},${g},${b},${0.18 * intensity}) 32%, transparent 62%)`
|
||||
el.style.transform = `scale(${scale})`
|
||||
el.style.opacity = String(0.5 + intensity * 0.5)
|
||||
}
|
||||
raf = requestAnimationFrame(tick)
|
||||
}
|
||||
raf = requestAnimationFrame(tick)
|
||||
return () => cancelAnimationFrame(raf)
|
||||
}, [audioLevel])
|
||||
|
||||
return <div ref={ref} className="aura" />
|
||||
}
|
||||
@@ -1,357 +0,0 @@
|
||||
import { Canvas, useFrame } from "@react-three/fiber"
|
||||
import { OrbitControls } from "@react-three/drei"
|
||||
import { useEffect, useRef, useState, type MutableRefObject } from "react"
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"
|
||||
import { Object3D, Vector3, AnimationMixer, LoopPingPong, LoopOnce, AdditiveBlending, DoubleSide, type AnimationAction } from "three"
|
||||
import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm"
|
||||
import { VRMAnimationLoaderPlugin, createVRMAnimationClip, type VRMAnimation } from "@pixiv/three-vrm-animation"
|
||||
import type { Emotion } from "../lib/voice/sentiment"
|
||||
|
||||
// 3D-Avatar (VRM). BEWEGUNG: echte Mocap-Clips (.vrma) je nach Zustand, weich ueberblendet
|
||||
// (Relax/LookAround im Leerlauf, Thinking beim Nachdenken). Darueber gelegt: Atmen + Sprech-Nicken
|
||||
// (additiv), Lippensync (echte Vokal-Mundformen), Blick (ohne Schielen), Mimik, Blinzeln.
|
||||
// Faellt sicher auf die alte prozedurale Bewegung zurueck, falls Clips nicht laden.
|
||||
|
||||
type LevelRef = MutableRefObject<{ current: number; aa?: number; ih?: number; ou?: number }>
|
||||
type EmotionRef = MutableRefObject<Emotion>
|
||||
type CursorRef = MutableRefObject<{ x: number; y: number }>
|
||||
type NumRef = MutableRefObject<number>
|
||||
|
||||
// Welche Clips geladen werden (Dateien unter public/vrma/, MIT-Lizenz, siehe ATTRIBUTION.md).
|
||||
// Die .vrma sind GESTEN (kein echter Ruhe-Loop) -> nur fuer klare Zustaende/Einschuebe nutzen,
|
||||
// nicht als Dauerschleife. Ruhiger Leerlauf laeuft prozedural. Weitere Clips liegen auf Platte bereit.
|
||||
const CLIPS = ["Thinking", "Relax", "LookAround"] as const
|
||||
const BREAK_CLIPS = ["Relax", "LookAround"] as const // gelegentliche Einmal-Einschuebe im Leerlauf
|
||||
|
||||
const REST: Record<string, [number, number, number]> = {
|
||||
leftUpperArm: [0, 0, 1.2], rightUpperArm: [0, 0, -1.2], leftLowerArm: [0, -0.2, 0], rightLowerArm: [0, 0.2, 0],
|
||||
}
|
||||
function setBone(vrm: VRM, name: string, x: number, y: number, z: number) {
|
||||
const b = vrm.humanoid?.getNormalizedBoneNode(name as any)
|
||||
if (b) b.rotation.set(x, y, z)
|
||||
}
|
||||
function addBone(vrm: VRM, name: string, x: number, y: number, z: number) {
|
||||
const b = vrm.humanoid?.getNormalizedBoneNode(name as any)
|
||||
if (b) { b.rotation.x += x; b.rotation.y += y; b.rotation.z += z }
|
||||
}
|
||||
function applyRestPose(vrm: VRM) {
|
||||
for (const [name, r] of Object.entries(REST)) setBone(vrm, name, r[0], r[1], r[2])
|
||||
vrm.humanoid?.update()
|
||||
}
|
||||
|
||||
// ADDITIVE Lebendigkeit OBEN AUF dem Clip: dezentes Atmen, Sprech-Nicken, Vorlehnen beim Zuhoeren.
|
||||
// Kleine Amplituden -> ergaenzt den Clip, kaempft nicht mit ihm.
|
||||
function addLife(vrm: VRM, t: number, speak: number, lean: number) {
|
||||
const breathe = Math.sin(t * 1.6)
|
||||
addBone(vrm, "spine", breathe * 0.012 + lean * 0.05, 0, 0)
|
||||
addBone(vrm, "chest", breathe * 0.01, 0, 0)
|
||||
const beat = Math.sin(t * 2.4) + Math.sin(t * 3.7) * 0.5
|
||||
const nod = speak * (0.018 * beat + 0.012 * Math.sin(t * 1.3))
|
||||
addBone(vrm, "neck", nod * 0.4 + lean * 0.03, 0, 0)
|
||||
addBone(vrm, "head", nod, 0, 0)
|
||||
}
|
||||
|
||||
// Fallback (keine Clips): die fruehere rein prozedurale Bewegung, leicht abgespeckt.
|
||||
function applyIdleProcedural(vrm: VRM, t: number, level: number, lean: number, speak: number) {
|
||||
const breathe = Math.sin(t * 1.6), sway = Math.sin(t * 0.45), weight = Math.sin(t * 0.32), weight2 = Math.sin(t * 0.21 + 1)
|
||||
const emph = Math.min(1, level * 1.6)
|
||||
const beat = Math.sin(t * 2.4) + Math.sin(t * 3.7) * 0.5
|
||||
const nod = (emph * 0.05 + speak * 0.02) * beat + emph * Math.sin(t * 1.3) * 0.03
|
||||
const tilt = Math.sin(t * 0.7 + 0.5) * 0.03 + speak * Math.sin(t * 0.9) * 0.045
|
||||
setBone(vrm, "hips", 0, weight * 0.06, weight * 0.04 + weight2 * 0.02)
|
||||
setBone(vrm, "spine", breathe * 0.03 + lean * 0.08 + speak * emph * 0.03, sway * 0.03, -weight * 0.04)
|
||||
setBone(vrm, "chest", breathe * 0.025 + lean * 0.025, sway * 0.022, weight2 * 0.012)
|
||||
setBone(vrm, "upperChest", breathe * 0.018, sway * 0.01, 0)
|
||||
setBone(vrm, "neck", nod * 0.5, 0, tilt * 0.5)
|
||||
setBone(vrm, "head", nod + Math.sin(t * 0.6) * 0.02, Math.sin(t * 0.27) * 0.035, tilt + Math.sin(t * 0.5) * 0.025)
|
||||
const armSwing = Math.sin(t * 0.8) * 0.05 + speak * Math.sin(t * 1.6) * 0.045
|
||||
const lift = speak * emph * 0.07
|
||||
setBone(vrm, "leftUpperArm", -lift, 0, 1.16 + armSwing + weight * 0.05)
|
||||
setBone(vrm, "rightUpperArm", -lift, 0, -1.16 - armSwing + weight * 0.05)
|
||||
setBone(vrm, "leftLowerArm", 0, -0.18 - Math.sin(t * 0.8) * 0.04 - speak * 0.07, 0)
|
||||
setBone(vrm, "rightLowerArm", 0, 0.18 + Math.sin(t * 0.8) * 0.04 + speak * 0.07, 0)
|
||||
}
|
||||
|
||||
const EXPRESSIONS = ["happy", "angry", "sad", "surprised", "relaxed"] as const
|
||||
const EMO_TO_EXPR: Record<Emotion, string | null> = {
|
||||
neutral: null, happy: "happy", angry: "angry", sad: "sad", surprised: "surprised", relaxed: "relaxed",
|
||||
}
|
||||
|
||||
function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onError }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef; status: MutableRefObject<string>
|
||||
cursor?: CursorRef; pat?: NumRef; dance?: NumRef; onError: (m: string) => void
|
||||
}) {
|
||||
const [vrm, setVrm] = useState<VRM | null>(null)
|
||||
const smooth = useRef<Record<string, number>>({})
|
||||
const blink = useRef({ t: 0, next: 3, active: 0 })
|
||||
const speak = useRef(0) // 0..1 Sprech-Lebendigkeit
|
||||
const lean = useRef(0) // 0..1 Vorlehnen (Zuhoeren/lauter Ton)
|
||||
const gaze = useRef(new Object3D()) // weit entfernter Blickpunkt (gegen Schielen bei naher Kamera)
|
||||
const headPos = useRef(new Vector3())
|
||||
const camPos = useRef(new Vector3())
|
||||
const camRight = useRef(new Vector3())
|
||||
const camUp = useRef(new Vector3())
|
||||
const patReact = useRef(0) // 0..1 Reaktion aufs Antippen/Streicheln
|
||||
const lastPat = useRef(0)
|
||||
// Bewegungs-Clips
|
||||
const mixer = useRef<AnimationMixer | null>(null)
|
||||
const actions = useRef<Record<string, AnimationAction>>({})
|
||||
const thinkW = useRef(0) // 0..1 Einblendung der Thinking-Pose
|
||||
const sac = useRef({ x: 0, y: 0, tx: 0, ty: 0, t: 0, next: 1.5 }) // Blick-Mikrobewegung (Sakkaden)
|
||||
const brk = useRef({ name: "", w: 0, t: 0, next: 15, active: false }) // gelegentlicher Bewegungs-Einschub
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false, loaded: VRM | null = null
|
||||
const loader = new GLTFLoader()
|
||||
loader.register((parser) => new VRMLoaderPlugin(parser))
|
||||
loader.load(url, (gltf) => {
|
||||
if (disposed) return
|
||||
const v = gltf.userData.vrm as VRM | undefined
|
||||
if (!v) { onError("Datei enthaelt kein gueltiges VRM-Modell."); return }
|
||||
VRMUtils.removeUnnecessaryVertices(gltf.scene)
|
||||
if (v.meta?.metaVersion === "0") VRMUtils.rotateVRM0(v)
|
||||
v.scene.rotation.y = Math.PI
|
||||
applyRestPose(v)
|
||||
loaded = v; setVrm(v)
|
||||
|
||||
// Bewegungs-Clips laden + Mixer aufsetzen (additiv-frei: voller Body-Override durch den Clip)
|
||||
const mx = new AnimationMixer(v.scene)
|
||||
mixer.current = mx
|
||||
const aLoader = new GLTFLoader()
|
||||
aLoader.register((parser) => new VRMAnimationLoaderPlugin(parser))
|
||||
for (const name of CLIPS) {
|
||||
aLoader.load(`/vrma/${name}.vrma`, (g) => {
|
||||
if (disposed) return
|
||||
const anims = g.userData.vrmAnimations as VRMAnimation[] | undefined
|
||||
if (!anims?.[0]) return
|
||||
const clip = createVRMAnimationClip(anims[0], v)
|
||||
const action = mx.clipAction(clip)
|
||||
action.timeScale = 0.85 // etwas ruhiger
|
||||
action.enabled = true
|
||||
action.setEffectiveWeight(0) // startet aus; blendet nur bei Bedarf ein
|
||||
if (name === "Thinking") { action.setLoop(LoopPingPong, Infinity); action.play() } // laeuft immer (Gewicht 0)
|
||||
else { action.setLoop(LoopOnce, 1); action.clampWhenFinished = true } // Einschub: einmal, erst bei Trigger
|
||||
actions.current[name] = action
|
||||
}, undefined, (e) => console.warn(`VRMA ${name} nicht geladen:`, e))
|
||||
}
|
||||
}, undefined, (err) => { console.error("VRM-Load:", err); onError("Avatar konnte nicht geladen werden.") })
|
||||
return () => {
|
||||
disposed = true
|
||||
mixer.current?.stopAllAction(); mixer.current = null; actions.current = {}
|
||||
if (loaded) VRMUtils.deepDispose(loaded.scene)
|
||||
setVrm(null)
|
||||
}
|
||||
}, [url, onError])
|
||||
|
||||
useFrame((state, delta) => {
|
||||
if (!vrm) return
|
||||
const st = status.current
|
||||
const tc = state.clock.elapsedTime
|
||||
const target = audioLevel.current?.current ?? 0
|
||||
const spk = (speak.current += (((st === "speaking") ? 1 : 0) - speak.current) * Math.min(1, delta * 4))
|
||||
const attentive = st === "listening" ? 1 : 0
|
||||
lean.current += ((Math.min(1, target * 1.6) + attentive * 0.5) - lean.current) * Math.min(1, delta * 3)
|
||||
|
||||
// Bewegung: ruhiger PROZEDURALER Leerlauf als Basis. Echte Clips blenden zustandsabhaengig ein:
|
||||
// Thinking beim Nachdenken (Dauer) + gelegentliche Einmal-Einschuebe im Leerlauf (Strecken/Umschauen
|
||||
// alle ~20-35s, dann zurueck in die Ruhe). Die .vrma sind Gesten -> nie als Dauerschleife.
|
||||
const mx = mixer.current
|
||||
const thinkAction = mx ? actions.current["Thinking"] : undefined
|
||||
const calm = st === "idle"
|
||||
const wantThink = (st === "thinking" || st === "transcribing") ? 1 : 0
|
||||
const br = brk.current
|
||||
// Einschub planen: nur wenn ruhig + keiner laeuft; Timer zuruecksetzen, sobald nicht mehr ruhig
|
||||
if (mx && !br.active) {
|
||||
if (calm) {
|
||||
br.t += delta
|
||||
if (br.t > br.next) {
|
||||
const pick = BREAK_CLIPS[Math.floor(Math.random() * BREAK_CLIPS.length)]
|
||||
const a = actions.current[pick]
|
||||
if (a) { a.reset(); a.play(); br.name = pick; br.active = true; br.w = 0 }
|
||||
br.t = 0; br.next = 20 + Math.random() * 15
|
||||
}
|
||||
} else { br.t = 0 }
|
||||
}
|
||||
thinkW.current += (wantThink - thinkW.current) * Math.min(1, delta * 4)
|
||||
thinkAction?.setEffectiveWeight(thinkW.current)
|
||||
// Einschub-Gewicht: ausblenden, wenn nicht mehr ruhig / Nachdenken startet / Clip fast zu Ende
|
||||
if (br.active) {
|
||||
const a = actions.current[br.name]
|
||||
const dur = a?.getClip().duration ?? 1
|
||||
const finishing = !calm || wantThink === 1 || (a ? a.time >= dur - 0.4 : true)
|
||||
br.w += ((finishing ? 0 : 1) - br.w) * Math.min(1, delta * 3)
|
||||
a?.setEffectiveWeight(br.w)
|
||||
if (finishing && br.w < 0.02) { a?.stop(); br.active = false; br.name = "" }
|
||||
}
|
||||
if (mx) mx.update(delta)
|
||||
if (thinkW.current > 0.05 || (br.active && br.w > 0.05)) {
|
||||
addLife(vrm, tc, spk, lean.current) // Clip-Pose + Atmen/Nicken obendrauf
|
||||
} else {
|
||||
applyIdleProcedural(vrm, tc, target, lean.current, spk) // ruhiger Leerlauf
|
||||
}
|
||||
// MateEngine-Idee: Kopf dreht sich dezent zum Mauszeiger (Augen folgen unten im lookAt-Block)
|
||||
const cur = cursor?.current
|
||||
if (cur) {
|
||||
const hx = Math.max(-1, Math.min(1, cur.x)), hy = Math.max(-1, Math.min(1, cur.y))
|
||||
addBone(vrm, "neck", -hy * 0.06, hx * 0.10, 0)
|
||||
addBone(vrm, "head", -hy * 0.08, hx * 0.14, 0)
|
||||
}
|
||||
// Antippen/Streicheln: kurze freudige Reaktion (Kopf-Wackeln; Laecheln in der Mimik unten)
|
||||
if (pat && pat.current !== lastPat.current) { lastPat.current = pat.current; patReact.current = 1 }
|
||||
patReact.current *= Math.max(0, 1 - delta * 1.4)
|
||||
if (patReact.current > 0.01) addBone(vrm, "head", -patReact.current * 0.05, 0, Math.sin(tc * 18) * patReact.current * 0.06)
|
||||
// Tanzen zur Musik: rhythmische Ganzkoerper-Bewegung, skaliert mit der Bass-Energie
|
||||
const dl = dance?.current ?? 0
|
||||
if (dl > 0.03) {
|
||||
const bt = tc * 5.5
|
||||
addBone(vrm, "hips", 0, Math.sin(bt * 0.5) * 0.14 * dl, Math.sin(bt) * 0.07 * dl)
|
||||
addBone(vrm, "spine", 0, Math.sin(bt * 0.5) * 0.06 * dl, Math.sin(bt) * 0.06 * dl)
|
||||
addBone(vrm, "chest", 0, 0, Math.sin(bt + 0.5) * 0.05 * dl)
|
||||
addBone(vrm, "head", Math.sin(bt) * 0.05 * dl, Math.sin(bt * 0.5) * 0.05 * dl, Math.sin(bt) * 0.05 * dl)
|
||||
addBone(vrm, "leftUpperArm", 0, 0, Math.sin(bt) * 0.3 * dl)
|
||||
addBone(vrm, "rightUpperArm", 0, 0, -Math.sin(bt) * 0.3 * dl)
|
||||
vrm.scene.position.y = Math.abs(Math.sin(bt)) * 0.05 * dl // Huepfen
|
||||
} else if (vrm.scene.position.y !== 0) {
|
||||
vrm.scene.position.y += (0 - vrm.scene.position.y) * Math.min(1, delta * 5)
|
||||
}
|
||||
|
||||
// Blick: weit entfernter Punkt in Kamerarichtung -> Augen parallel (kein Schielen), wirkt "auf dich".
|
||||
if (vrm.lookAt) {
|
||||
const headNode = vrm.humanoid?.getNormalizedBoneNode("head")
|
||||
const hp = headPos.current
|
||||
if (headNode) headNode.getWorldPosition(hp); else hp.set(0, 1.3, 0)
|
||||
const cp = camPos.current.copy(state.camera.position)
|
||||
gaze.current.position.copy(cp).sub(hp).multiplyScalar(4).add(hp)
|
||||
// Augen folgen dem Mauszeiger: Blickpunkt in Kamera-Rechts/Hoch-Richtung verschieben
|
||||
if (cur) {
|
||||
const e = state.camera.matrixWorld.elements
|
||||
camRight.current.set(e[0], e[1], e[2]).normalize()
|
||||
camUp.current.set(e[4], e[5], e[6]).normalize()
|
||||
const cx = Math.max(-1.5, Math.min(1.5, cur.x)), cy = Math.max(-1.5, Math.min(1.5, cur.y))
|
||||
gaze.current.position.addScaledVector(camRight.current, cx * 1.8)
|
||||
gaze.current.position.addScaledVector(camUp.current, cy * 1.8)
|
||||
}
|
||||
// dezente Blick-Mikrobewegung (Sakkaden) -> das Gesicht wirkt nicht eingefroren
|
||||
const s = sac.current
|
||||
s.t += delta
|
||||
if (s.t > s.next) {
|
||||
s.tx = (Math.random() - 0.5) * 0.22; s.ty = (Math.random() - 0.5) * 0.12
|
||||
s.t = 0; s.next = 0.8 + Math.random() * 2.4
|
||||
}
|
||||
s.x += (s.tx - s.x) * Math.min(1, delta * 10)
|
||||
s.y += (s.ty - s.y) * Math.min(1, delta * 10)
|
||||
gaze.current.position.x += s.x; gaze.current.position.y += s.y
|
||||
vrm.lookAt.target = gaze.current
|
||||
}
|
||||
|
||||
const em = vrm.expressionManager
|
||||
if (em) {
|
||||
// Lippensync v2: echte Vokal-Mundformen (aa/ih/ou) aus der Audio-Analyse statt nur "Mund auf".
|
||||
const lv = audioLevel.current
|
||||
const visTargets: Record<string, number> = { aa: lv?.aa ?? target, ih: lv?.ih ?? 0, ou: lv?.ou ?? 0 }
|
||||
for (const v of ["aa", "ih", "ou"]) {
|
||||
const cv = smooth.current[v] ?? 0
|
||||
const nv = cv + (visTargets[v] - cv) * Math.min(1, delta * 14)
|
||||
smooth.current[v] = nv; em.setValue(v, nv)
|
||||
}
|
||||
const want = EMO_TO_EXPR[emotion.current]
|
||||
for (const name of EXPRESSIONS) {
|
||||
const tv = want === name ? 0.75 : 0
|
||||
const cv = smooth.current[name] ?? 0
|
||||
const nv = cv + (tv - cv) * Math.min(1, delta * 4)
|
||||
smooth.current[name] = nv; em.setValue(name, nv)
|
||||
}
|
||||
// kurze passende Mimik waehrend eines Bewegungs-Einschubs (Laecheln beim Strecken)
|
||||
if (brk.current.active) {
|
||||
const expr = brk.current.name === "Relax" ? "happy" : "relaxed"
|
||||
em.setValue(expr, Math.max(smooth.current[expr] ?? 0, brk.current.w * 0.35))
|
||||
}
|
||||
// Streichel-Reaktion: Laecheln
|
||||
if (patReact.current > 0.01) em.setValue("happy", Math.max(smooth.current.happy ?? 0, patReact.current * 0.85))
|
||||
const b = blink.current
|
||||
b.t += delta
|
||||
if (b.active <= 0 && b.t > b.next) { b.active = 0.16; b.t = 0; b.next = 3 + Math.random() * 4 }
|
||||
let blinkVal = 0
|
||||
if (b.active > 0) { b.active -= delta; const p = 1 - b.active / 0.16; blinkVal = 1 - Math.abs(p - 0.5) * 2 }
|
||||
em.setValue("blink", Math.max(0, blinkVal))
|
||||
}
|
||||
vrm.update(delta)
|
||||
})
|
||||
|
||||
return vrm ? <primitive object={vrm.scene} /> : null
|
||||
}
|
||||
|
||||
// Hologramm-Beiwerk: Projektor-Sockel (Glow-Ringe am Boden) + schwebender Kristall daneben.
|
||||
// Reine Additiv-Meshes -> berühren die VRM-Materialien NICHT (voll reversibel per Toggle).
|
||||
function HoloRig() {
|
||||
const ringA = useRef<any>(null), ringB = useRef<any>(null), crystal = useRef<any>(null)
|
||||
useFrame((s, d) => {
|
||||
const t = s.clock.elapsedTime
|
||||
if (ringA.current) ringA.current.rotation.z += d * 0.3
|
||||
if (ringB.current) ringB.current.rotation.z -= d * 0.55
|
||||
if (crystal.current) {
|
||||
crystal.current.rotation.y += d * 1.1
|
||||
crystal.current.rotation.x = Math.sin(t * 0.8) * 0.3
|
||||
crystal.current.position.y = 1.18 + Math.sin(t * 1.6) * 0.04
|
||||
}
|
||||
})
|
||||
const cyan = "#4fd8ff"
|
||||
return (
|
||||
<group>
|
||||
{/* Projektor-Sockel: weicher Glow-Disc + zwei rotierende Ringe am Boden (Fuesse bei y=0) */}
|
||||
<group position={[0, 0.02, 0]} rotation={[-Math.PI / 2, 0, 0]}>
|
||||
<mesh><ringGeometry args={[0, 0.36, 48]} /><meshBasicMaterial color={cyan} transparent opacity={0.07} blending={AdditiveBlending} side={DoubleSide} depthWrite={false} /></mesh>
|
||||
<mesh ref={ringA}><ringGeometry args={[0.3, 0.36, 64]} /><meshBasicMaterial color={cyan} transparent opacity={0.85} blending={AdditiveBlending} side={DoubleSide} depthWrite={false} /></mesh>
|
||||
<mesh ref={ringB}><ringGeometry args={[0.19, 0.215, 48]} /><meshBasicMaterial color={cyan} transparent opacity={0.6} blending={AdditiveBlending} side={DoubleSide} depthWrite={false} /></mesh>
|
||||
</group>
|
||||
{/* schwebender Kristall neben ihr (Schulterhoehe) */}
|
||||
<mesh ref={crystal} position={[0.44, 1.18, 0]} scale={0.075}>
|
||||
<octahedronGeometry args={[1, 0]} />
|
||||
<meshBasicMaterial color={cyan} transparent opacity={0.85} blending={AdditiveBlending} depthWrite={false} />
|
||||
</mesh>
|
||||
</group>
|
||||
)
|
||||
}
|
||||
|
||||
export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: {
|
||||
url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string
|
||||
cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean
|
||||
}) {
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
const [holo, setHolo] = useState(() => localStorage.getItem("lucy_holo") !== "0") // Hologramm-Look, Standard: an
|
||||
const statusRef = useRef(status); statusRef.current = status
|
||||
const toggleHolo = () => setHolo((h) => { const n = !h; localStorage.setItem("lucy_holo", n ? "1" : "0"); return n })
|
||||
return (
|
||||
<div className={`avatarCanvas${holo ? " holo" : ""}`} style={{ position: "relative", height: "100%", width: "100%" }}>
|
||||
<Canvas camera={{ position: [0, 1.35, 1.25], fov: 30 }} dpr={[1, 1.5]}
|
||||
gl={{ alpha: true, antialias: true, powerPreference: "high-performance" }} style={{ background: "transparent" }}>
|
||||
{holo ? (
|
||||
<>
|
||||
<ambientLight intensity={0.55} color="#9fe6ff" />
|
||||
<directionalLight position={[0, 2, -2.5]} intensity={2.4} color="#3fd4ff" />{/* Rim von hinten -> Silhouetten-Glow */}
|
||||
<directionalLight position={[1.5, 1.5, 2]} intensity={0.5} color="#cceeff" />
|
||||
<HoloRig />
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ambientLight intensity={0.85} />
|
||||
<directionalLight position={[1, 2, 2]} intensity={1.1} />
|
||||
<directionalLight position={[-1, 1, -1]} intensity={0.4} />
|
||||
</>
|
||||
)}
|
||||
<VrmModel key={url} url={url} audioLevel={audioLevel} emotion={emotion} status={statusRef} cursor={cursor} pat={pat} dance={dance} onError={setErr} />
|
||||
{controls && (
|
||||
<OrbitControls target={[0, 1.3, 0]} enablePan={false} minDistance={0.7} maxDistance={3}
|
||||
minPolarAngle={Math.PI / 3} maxPolarAngle={Math.PI / 1.8} />
|
||||
)}
|
||||
</Canvas>
|
||||
{holo && (<><div className="holoTint" /><div className="holoScan" /></>)}
|
||||
<button className={`holoToggle${holo ? " on" : ""}`} onClick={toggleHolo} title="Hologramm-Look an/aus">◇</button>
|
||||
{err && (
|
||||
<div style={{ position: "absolute", left: 0, right: 0, bottom: 12, margin: "0 auto", width: "fit-content",
|
||||
borderRadius: 8, background: "rgba(239,68,68,0.15)", border: "1px solid rgba(239,68,68,0.3)",
|
||||
padding: "6px 12px", fontSize: 12, color: "#fca5a5" }}>{err}</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
import { useState, type ReactNode } from "react"
|
||||
import ReactMarkdown from "react-markdown"
|
||||
import remarkGfm from "remark-gfm"
|
||||
|
||||
// Chat-Markdown fuer Lucys Antworten: Code-Bloecke mit Kopier-Knopf, Links oeffnen im System-Browser,
|
||||
// gaengige Formatierung (fett/listen/inline-code). Die STIMME liest davon nichts vor — cleanForTTS
|
||||
// filtert Code/Links vorm Sprechen; hier geht es rein um die lesbare/kopierbare Anzeige.
|
||||
|
||||
function CopyBtn({ text, label = "Kopieren" }: { text: string; label?: string }) {
|
||||
const [done, setDone] = useState(false)
|
||||
return (
|
||||
<button
|
||||
className="copyBtn"
|
||||
onClick={async () => {
|
||||
try { await navigator.clipboard.writeText(text); setDone(true); setTimeout(() => setDone(false), 1200) } catch { /* */ }
|
||||
}}
|
||||
>{done ? "✓ Kopiert" : label}</button>
|
||||
)
|
||||
}
|
||||
|
||||
function nodeText(children: ReactNode): string {
|
||||
if (children == null) return ""
|
||||
if (typeof children === "string" || typeof children === "number") return String(children)
|
||||
if (Array.isArray(children)) return children.map(nodeText).join("")
|
||||
// @ts-expect-error – React-Element-Kinder
|
||||
if (children?.props?.children) return nodeText(children.props.children)
|
||||
return ""
|
||||
}
|
||||
|
||||
export function ChatMarkdown({ text }: { text: string }) {
|
||||
return (
|
||||
<div className="md">
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
a: ({ href, children }) => (
|
||||
<a className="mdLink" href={href}
|
||||
onClick={(e) => { e.preventDefault(); if (href) void window.lucy?.openExternal(href) }}>
|
||||
{children}
|
||||
</a>
|
||||
),
|
||||
// pre durchreichen -> der Code-Block baut seinen eigenen Container (kein <div> in <pre>)
|
||||
pre: ({ children }) => <>{children}</>,
|
||||
code: ({ className, children, ...props }) => {
|
||||
const raw = nodeText(children).replace(/\n$/, "")
|
||||
const isBlock = /language-/.test(className || "") || raw.includes("\n")
|
||||
if (!isBlock) return <code className="mdInlineCode" {...props}>{children}</code>
|
||||
const lang = (className || "").replace(/language-/, "") || "code"
|
||||
return (
|
||||
<div className="codeBlock">
|
||||
<div className="codeBar"><span className="codeLang">{lang}</span><CopyBtn text={raw} /></div>
|
||||
<pre><code className={className}>{children}</code></pre>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
}}
|
||||
>{text}</ReactMarkdown>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
// Zentrale Endpunkte. Hirn + STT laufen auf der BOX (wie die WebUI), die Stimme LOKAL (pocket-tts, CPU).
|
||||
export const BOX_URL = "http://192.168.178.151:9001" // MC2-Backend: /api/voice/stt, /api/voice/chat
|
||||
export const TTS_URL = "http://127.0.0.1:8130" // lokaler Lucy-TTS (pocket-tts, vom Main-Prozess gespawnt)
|
||||
|
||||
// Persona/Anrede: Lucy spricht den Nutzer als "Commander" an. ECHTE Umlaute erzwingen (sonst klingt TTS grausam).
|
||||
export const SYSTEM_PROMPT =
|
||||
"Du bist Lucy, eine gesprochene Assistentin, und redest den Nutzer mit „Commander“ an. " +
|
||||
"Antworte natürlich, locker und mit etwas Persönlichkeit — du darfst ruhig ein bisschen chatty und frech sein, " +
|
||||
"aber bleib FOKUSSIERT: in der Regel 2 bis 3 Sätze. Beantworte die Frage direkt (gern mit einer kleinen menschlichen " +
|
||||
"Note) und hol dann nicht unnötig aus. Keine ungefragten Meta-Kommentare, Warnungen oder Wiederholungen. " +
|
||||
"Ausführlich nur, wenn ausdrücklich gewünscht. " +
|
||||
"TOOLS: Du DARFST und SOLLST Tools/Skills nutzen, wenn sie wirklich helfen (Live-Daten wie Wetter, " +
|
||||
"Gedächtnis-/Gehirn-Zugriff, echte Analyse). Aber NICHT für Triviales, das du eh weißt (Uhrzeit, " +
|
||||
"Smalltalk, Allgemeinwissen) — da antworte direkt. Ketten NIEMALS mehrere Tools wild aneinander oder " +
|
||||
"probier herum (kein SSH + Screenshot + Shell für eine simple Info). Wenn ein Tool länger dauert, sag " +
|
||||
"kurz Bescheid, bevor du es nutzt (z. B. „Moment, das schau ich kurz nach, Commander.“), dann liefere das Ergebnis. " +
|
||||
"In Terminal-Befehlen IMMER vollständige URLs mit https:// schreiben (z. B. https://wttr.in/...) und bei " +
|
||||
"Netz-Abrufen ein kurzes Timeout setzen (curl: --max-time 8) — sonst hängt der Befehl oder wird vom " +
|
||||
"Sicherheits-Scan aufgehalten. " +
|
||||
"WICHTIG für die Sprachausgabe: Beginne mit einem KURZEN ersten Satz (nur wenige Wörter, z. B. " +
|
||||
"„Na klar, Commander!“). Fasse dich kurz und komm auf den Punkt — JEDES Satzzeichen (Komma wie Punkt) " +
|
||||
"wird als hörbare Pause gesprochen, je weniger Wörter und Satzzeichen, desto flüssiger. Vermeide vor " +
|
||||
"allem lange Komma-Ketten und Schachtelsätze; zwei, drei knappe Aussagen reichen. Also lieber " +
|
||||
"„Das Backup lief durch. Keine Fehler. Alles stabil.“ statt „Das Backup ist durchgelaufen, es gab " +
|
||||
"keine Fehler, und alles läuft stabil.“. Kurze Sätze starten sofort hörbar und klingen sauberer. " +
|
||||
"Halte den gesprochenen Teil in reinem Fließtext (keine Aufzählungszeichen, keine Emojis). " +
|
||||
"NUR wenn der Commander ausdrücklich nach Code, Befehlen oder einem Link fragt, gib diese im Text aus — " +
|
||||
"Code in Markdown-Codeblöcken (```), Links als vollständige URL. Diese werden angezeigt, aber NICHT vorgelesen; " +
|
||||
"sprich dann nur eine kurze Einleitung dazu (z. B. „Hier ist das Skript, Commander.“). " +
|
||||
"Verwende IMMER echte deutsche Umlaute (ä, ö, ü, ß) und NIEMALS ae, oe, ue oder ss als Ersatz. " +
|
||||
"Schreibe Zahlen, Modellbezeichnungen und Abkürzungen EXAKT und normal (z. B. „RX 9070 XT“, „4 GB“, " +
|
||||
"„25 Grad“) — schreibe Ziffern NIEMALS als Wörter aus und interpretiere sie nicht als Komma-/Dezimalzahlen. " +
|
||||
"Nenne beim Sprechen KEINE Zeitzonen-Codes (UTC, CET, GMT o.ä.) und keine technischen Zeit-Zusätze — " +
|
||||
"sag einfach die lokale Uhrzeit natürlich (z. B. „Es ist 19 Uhr 21, Commander.“). " +
|
||||
"Du kannst optional GANZ AM ENDE deiner Antwort einen Stimmungs-Tag anhängen: <emo:happy>, <emo:sad>, " +
|
||||
"<emo:angry>, <emo:surprised>, <emo:relaxed> oder <emo:neutral>. Er wird nicht angezeigt oder vorgelesen " +
|
||||
"und steuert nur deinen Gesichtsausdruck — wähle ihn passend zum Inhalt deiner Antwort."
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
lucy?: {
|
||||
minimize: () => Promise<void>
|
||||
close: () => Promise<void>
|
||||
setMode: (m: "full" | "overlay") => Promise<"full" | "overlay">
|
||||
getMode: () => Promise<"full" | "overlay">
|
||||
togglePin: () => Promise<boolean>
|
||||
onMode: (cb: (m: "full" | "overlay") => void) => () => void
|
||||
onHotkey: (cb: () => void) => () => void
|
||||
onCursor: (cb: (p: { x: number; y: number }) => void) => () => void
|
||||
captureScreen: () => Promise<string[]>
|
||||
listWindows: () => Promise<{ id: string; name: string }[]>
|
||||
captureWindow: (id: string) => Promise<string | null>
|
||||
captureByName: (q: string) => Promise<{ name: string; image: string } | null>
|
||||
openExternal: (url: string) => Promise<void>
|
||||
setClickThrough: (on: boolean) => Promise<boolean>
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
// Audio-Wiedergabe + Pegelmessung fuers Lippensync (Avatar liest level.current).
|
||||
// Zwei Wege: enqueue(WAV-Buffer) [z.B. Warmup] und playPcmStream(PCM16-Stream) [die Live-Antwort,
|
||||
// lueckenlos via Web-Audio-Scheduling, niedrige Time-to-first-audio].
|
||||
//
|
||||
// LIPPENSYNC v2: statt nur einem Pegel ("Mund auf/zu") leiten wir aus dem Frequenz-Spektrum
|
||||
// echte Vokal-MUNDFORMEN ab (level.aa/ih/ou). Idee: die Helligkeit des Klangs (spektraler
|
||||
// Schwerpunkt) verraet grob den Vokal — helle Laute (i/e) = breiter Mund, dunkle (o/u) = runder
|
||||
// Mund, mittig = offenes "a". Das ist nicht phonetisch exakt, sieht aber lebendig+passend aus.
|
||||
function smoothstep(a: number, b: number, x: number): number {
|
||||
const t = Math.max(0, Math.min(1, (x - a) / (b - a)))
|
||||
return t * t * (3 - 2 * t)
|
||||
}
|
||||
|
||||
export class AudioQueue {
|
||||
private ctx: AudioContext
|
||||
private analyser: AnalyserNode
|
||||
private gain: GainNode
|
||||
private queue: ArrayBuffer[] = []
|
||||
private playing = false
|
||||
private raf = 0
|
||||
private freq: Uint8Array
|
||||
// aktiver PCM-Stream (fuer Barge-in/clear): geplante Quellen + Abbruchsignal
|
||||
private streamSources: AudioBufferSourceNode[] = []
|
||||
private streamCancelled = false
|
||||
// current = Gesamt-Pegel (Aura/Bewegung); aa/ih/ou = Mundform-Gewichte fuer den Avatar
|
||||
readonly level = { current: 0, aa: 0, ih: 0, ou: 0 }
|
||||
onSpeaking?: (speaking: boolean) => void
|
||||
|
||||
static readVolume(): number {
|
||||
const raw = localStorage.getItem("lucy_volume")
|
||||
if (raw === null || raw === "") return 0.8
|
||||
const v = Number(raw)
|
||||
return Number.isNaN(v) ? 0.8 : Math.max(0, Math.min(1.5, v))
|
||||
}
|
||||
|
||||
constructor() {
|
||||
const Ctor = window.AudioContext || (window as any).webkitAudioContext
|
||||
this.ctx = new Ctor()
|
||||
this.analyser = this.ctx.createAnalyser()
|
||||
this.analyser.fftSize = 256
|
||||
this.analyser.smoothingTimeConstant = 0.6
|
||||
this.gain = this.ctx.createGain()
|
||||
this.gain.gain.value = AudioQueue.readVolume()
|
||||
this.analyser.connect(this.gain)
|
||||
this.gain.connect(this.ctx.destination)
|
||||
this.freq = new Uint8Array(this.analyser.frequencyBinCount)
|
||||
window.addEventListener("lucy-volume", (e: Event) => {
|
||||
const v = Number((e as CustomEvent).detail)
|
||||
if (!Number.isNaN(v)) this.gain.gain.value = Math.max(0, Math.min(1.5, v))
|
||||
})
|
||||
}
|
||||
|
||||
async enqueue(buf: ArrayBuffer) {
|
||||
this.queue.push(buf)
|
||||
if (!this.playing) await this.playNext()
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.queue = []
|
||||
// laufenden Stream stoppen (Barge-in)
|
||||
this.streamCancelled = true
|
||||
for (const s of this.streamSources) { try { s.stop() } catch { /* */ } }
|
||||
this.streamSources = []
|
||||
}
|
||||
|
||||
// Spielt einen fortlaufenden PCM16-mono-Stream lueckenlos ab: jede Frame-Charge wird auf der
|
||||
// Audio-Uhr direkt hinter die vorige geplant (kein onended-Gap). LEAD_IN puffert gegen Underruns.
|
||||
async playPcmStream(stream: ReadableStream<Uint8Array>, sampleRate: number): Promise<void> {
|
||||
if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } }
|
||||
this.streamCancelled = false
|
||||
this.streamSources = []
|
||||
const reader = stream.getReader()
|
||||
const LEAD_IN = 0.35 // Startpuffer: mehr Vorlauf -> Generierung bleibt vor der Wiedergabe (weniger Unterläufe/Stocken)
|
||||
let nextTime = 0, started = false, leftoverByte = -1, lastEnd = 0
|
||||
this.onSpeaking?.(true)
|
||||
this.startMeter()
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done || this.streamCancelled) break
|
||||
if (!value || value.length === 0) continue
|
||||
// ungerades Rest-Byte der vorigen Charge voranstellen, damit Int16-Frames sauber bleiben
|
||||
let bytes: Uint8Array
|
||||
if (leftoverByte >= 0) {
|
||||
bytes = new Uint8Array(value.length + 1)
|
||||
bytes[0] = leftoverByte
|
||||
bytes.set(value, 1)
|
||||
} else {
|
||||
bytes = value
|
||||
}
|
||||
const usable = bytes.length - (bytes.length % 2)
|
||||
leftoverByte = usable < bytes.length ? bytes[bytes.length - 1] : -1
|
||||
if (usable === 0) continue
|
||||
// in ein eigenes, 2-Byte-ausgerichtetes Buffer kopieren (value.byteOffset evtl. ungerade)
|
||||
const aligned = new Uint8Array(usable)
|
||||
aligned.set(bytes.subarray(0, usable))
|
||||
const i16 = new Int16Array(aligned.buffer)
|
||||
const f32 = new Float32Array(i16.length)
|
||||
for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 32768
|
||||
const audioBuf = this.ctx.createBuffer(1, f32.length, sampleRate)
|
||||
audioBuf.copyToChannel(f32, 0)
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
if (!started) { nextTime = this.ctx.currentTime + LEAD_IN; started = true }
|
||||
if (nextTime < this.ctx.currentTime) nextTime = this.ctx.currentTime + 0.02 // Underrun-Schutz
|
||||
src.start(nextTime)
|
||||
this.streamSources.push(src)
|
||||
src.onended = () => {
|
||||
const i = this.streamSources.indexOf(src)
|
||||
if (i >= 0) this.streamSources.splice(i, 1)
|
||||
}
|
||||
nextTime += audioBuf.duration
|
||||
lastEnd = nextTime
|
||||
}
|
||||
} finally {
|
||||
try { reader.releaseLock() } catch { /* */ }
|
||||
}
|
||||
// bis zum Ende der letzten geplanten Charge warten (sofern nicht abgebrochen)
|
||||
if (!this.streamCancelled) {
|
||||
const waitMs = Math.max(0, (lastEnd - this.ctx.currentTime) * 1000)
|
||||
await new Promise((r) => setTimeout(r, waitMs + 60))
|
||||
}
|
||||
this.onSpeaking?.(false)
|
||||
this.stopMeter()
|
||||
}
|
||||
|
||||
private async playNext(): Promise<void> {
|
||||
const buf = this.queue.shift()
|
||||
if (!buf) { this.playing = false; this.stopMeter(); this.onSpeaking?.(false); return }
|
||||
this.playing = true
|
||||
this.onSpeaking?.(true)
|
||||
if (this.ctx.state === "suspended") { try { await this.ctx.resume() } catch { /* */ } }
|
||||
let audioBuf: AudioBuffer
|
||||
try { audioBuf = await this.ctx.decodeAudioData(buf.slice(0)) } catch { return this.playNext() }
|
||||
const src = this.ctx.createBufferSource()
|
||||
src.buffer = audioBuf
|
||||
src.connect(this.analyser)
|
||||
src.onended = () => { void this.playNext() }
|
||||
src.start()
|
||||
this.startMeter()
|
||||
}
|
||||
|
||||
private startMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
const tick = () => {
|
||||
this.analyser.getByteFrequencyData(this.freq as any)
|
||||
const n = Math.min(this.freq.length, 48)
|
||||
let sum = 0, wsum = 0
|
||||
for (let i = 2; i < n; i++) { const m = this.freq[i]; sum += m; wsum += m * i }
|
||||
const avg = sum / (n - 2) / 255
|
||||
const open = Math.min(1, avg * 1.9)
|
||||
this.level.current = open
|
||||
// Spektraler Schwerpunkt 0..1 (wo sitzt die Klang-Energie) -> grobe Vokal-Form
|
||||
const c = sum > 0 ? wsum / sum : 2
|
||||
const cN = Math.max(0, Math.min(1, (c - 2) / (n - 2)))
|
||||
const bright = smoothstep(0.42, 0.72, cN) // hell -> i/e (breiter Mund)
|
||||
const dark = 1 - smoothstep(0.22, 0.5, cN) // dunkel -> o/u (runder Mund)
|
||||
const ih = open * bright
|
||||
const ou = open * dark
|
||||
const aa = open * (1 - Math.max(bright, dark) * 0.85) // sonst offenes "a"
|
||||
// leichte Glaettung gegen Flackern
|
||||
this.level.aa += (aa - this.level.aa) * 0.5
|
||||
this.level.ih += (ih - this.level.ih) * 0.5
|
||||
this.level.ou += (ou - this.level.ou) * 0.5
|
||||
this.raf = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
}
|
||||
|
||||
private stopMeter() {
|
||||
cancelAnimationFrame(this.raf)
|
||||
this.level.current = 0; this.level.aa = 0; this.level.ih = 0; this.level.ou = 0
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
// Perf-Diagnose (Client): Zeit pro Stufe ab „Mikro losgelassen". Opt-in via localStorage lucy_perf=1.
|
||||
// NUR Console (kein POST an den pocket_server — der generiert Audio; POSTs mittendrin
|
||||
// verursachten hakelige Sprachausgabe). (Aus useVoiceAgent extrahiert, Review P2-13a.)
|
||||
|
||||
export const PERF = typeof localStorage !== "undefined" && localStorage.getItem("lucy_perf") === "1"
|
||||
|
||||
export function plogSend(msg: string) { if (PERF) console.log(`[lucy-perf] ${msg}`) }
|
||||
export function plog(label: string, ms: number) { plogSend(`${label} = ${Math.round(ms)}ms`) }
|
||||
@@ -1,14 +0,0 @@
|
||||
// Leichtgewichtige Stimmungs-Heuristik → treibt Lucys Mimik. Regelbasiert (kein Modell).
|
||||
export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed"
|
||||
|
||||
const RULES: [Emotion, RegExp][] = [
|
||||
["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha)/i],
|
||||
["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i],
|
||||
["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i],
|
||||
["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i],
|
||||
]
|
||||
|
||||
export function sentimentToEmotion(text: string): Emotion {
|
||||
for (const [emo, rx] of RULES) if (rx.test(text)) return emo
|
||||
return "neutral"
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Pure Text-Verarbeitung für den Voice-Loop (aus useVoiceAgent extrahiert, Review P2-13a).
|
||||
// Alles hier ist zustandslos und einzeln testbar — der Hook orchestriert nur noch.
|
||||
|
||||
// Hermes schreibt manchmal ASCII-Umlaute (ue/ae/oe/ss) statt ä/ö/ü/ß -> pocket-tts liest die falsch vor.
|
||||
// Gezielt häufige UMLAUT-Stämme zurückwandeln. KONSERVATIV: nur Muster, bei denen ASCII fast immer ein
|
||||
// Umlaut ist (echte "ue"-Wörter wie aktuell/neue/Quelle bleiben unangetastet — die sind hier NICHT gelistet).
|
||||
const UMLAUT_FIX: [RegExp, string][] = [
|
||||
[/ueber/gi, "über"], [/\bfuer\b/gi, "für"], [/natuerlich/gi, "natürlich"], [/zurueck/gi, "zurück"],
|
||||
[/unterstuetz/gi, "unterstütz"], [/ueberpruef/gi, "überprüf"], [/\bpruef/gi, "prüf"], [/gefuehl/gi, "gefühl"],
|
||||
[/\bfuehl/gi, "fühl"], [/\bfuehr/gi, "führ"], [/\bfuenf/gi, "fünf"], [/\bgruen/gi, "grün"], [/\bfrueh/gi, "früh"],
|
||||
[/\bglueck/gi, "glück"], [/\bstueck/gi, "stück"], [/gemuetlich/gi, "gemütlich"], [/verfueg/gi, "verfüg"],
|
||||
[/\bmoecht/gi, "möcht"], [/\bkoenn/gi, "könn"], [/\bwuerd/gi, "würd"], [/\bmuess/gi, "müss"], [/\bduerf/gi, "dürf"],
|
||||
[/\bhaett/gi, "hätt"], [/\bwaer/gi, "wär"], [/\bspaet/gi, "spät"],
|
||||
[/\btaeglich/gi, "täglich"], [/\bnaechst/gi, "nächst"], [/\baehnlich/gi, "ähnlich"], [/waehrend/gi, "während"],
|
||||
[/\bwaehl/gi, "wähl"], [/erklaer/gi, "erklär"], [/\bschoen/gi, "schön"], [/\bgroess/gi, "größ"], [/\bhoer/gi, "hör"],
|
||||
[/\boeffn/gi, "öffn"], [/\bboese/gi, "böse"], [/\bloesch/gi, "lösch"], [/\bstoer/gi, "stör"], [/koennt/gi, "könnt"],
|
||||
// erweitert (30.06.): weitere häufige Stämme + sicheres -ität-Suffix. KONSERVATIV gewählt;
|
||||
// der TTS-Server (_fix_umlauts) hat zusätzlich ein Netz. neu/aktuell/Steuer/Feuer bleiben unberührt.
|
||||
[/itaet/gi, "ität"],
|
||||
[/\bmoeglich/gi, "möglich"], [/gespraech/gi, "gespräch"], [/\bmaerz/gi, "märz"],
|
||||
[/geschaeft/gi, "geschäft"], [/gefaehr/gi, "gefähr"], [/\bhaeng/gi, "häng"], [/\blaeng/gi, "läng"],
|
||||
[/\bmaenner/gi, "männer"], [/\bmaedchen/gi, "mädchen"], [/\bvaeter/gi, "väter"], [/\btraeum/gi, "träum"],
|
||||
[/\bsaetz/gi, "sätz"], [/\bplaetz/gi, "plätz"], [/\bkaelte/gi, "kälte"], [/\bzaehl/gi, "zähl"],
|
||||
[/\baerg/gi, "ärg"], [/\baerzt/gi, "ärzt"], [/aeusser/gi, "äußer"],
|
||||
[/\btuer/gi, "tür"], [/\bkueche/gi, "küche"], [/buech/gi, "büch"], [/\bbuero/gi, "büro"], [/\bbuerg/gi, "bürg"],
|
||||
[/\bsued/gi, "süd"], [/schueler/gi, "schüler"], [/uebung/gi, "übung"], [/kuenstl/gi, "künstl"],
|
||||
[/\bmuede/gi, "müde"], [/\bmuell/gi, "müll"], [/\bdrueck/gi, "drück"], [/stuetz/gi, "stütz"],
|
||||
[/\bmoebel/gi, "möbel"], [/loes/gi, "lös"], [/voellig/gi, "völlig"], [/zwoelf/gi, "zwölf"],
|
||||
[/\bkoenig/gi, "könig"], [/\bhoeh/gi, "höh"], [/\bdoerf/gi, "dörf"], [/\bwoert/gi, "wört"],
|
||||
]
|
||||
export function restoreUmlauts(s: string): string {
|
||||
let out = s
|
||||
for (const [re, rep] of UMLAUT_FIX) out = out.replace(re, rep as string)
|
||||
return out
|
||||
}
|
||||
|
||||
// Optionaler Stimmungs-Tag, den Hermes ans Ende haengen kann (<emo:happy> ...). Steuert NUR den
|
||||
// Gesichtsausdruck -> wird vor Anzeige UND vor dem Vorlesen entfernt. Fallback = Sentiment-Heuristik.
|
||||
export const EMO_TAG = /<emo:(happy|sad|angry|surprised|relaxed|neutral)>/i
|
||||
export function stripEmoTag(s: string): string { return s.replace(/<emo:\w+>/gi, "").trimEnd() }
|
||||
|
||||
// Hermes hängt bei selbst erzeugten Medien (z.B. eigener Screenshot via pc-control) einen Roh-Token
|
||||
// „MEDIA:<datei>" an. Das ist weder Sprech- noch Anzeigetext -> überall entfernen (bis Medien echt
|
||||
// gerendert werden). Betrifft auch Lucys eigene Bildschirm-Sicht, die den Screenshot ohnehin schon liefert.
|
||||
export function stripMedia(s: string): string {
|
||||
return s.replace(/\bMEDIA:\S+/gi, "").replace(/[ \t]{2,}/g, " ").replace(/[ \t]+\n/g, "\n").trimEnd()
|
||||
}
|
||||
|
||||
export function cleanForTTS(s: string): string {
|
||||
return restoreUmlauts(s)
|
||||
.replace(/```[\s\S]*?```/g, " ").replace(/`([^`]*)`/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/\bMEDIA:\S+/gi, " ")
|
||||
// Zeitzonen-Codes (UTC/CET/GMT…) + evtl. angehängte Zeit: Pocket mangelt die Akronyme zu
|
||||
// Kauderwelsch -> aus der Stimme entfernen (Anzeige behält sie). Lucy sagt die lokale Zeit ohnehin.
|
||||
.replace(/\b(?:UTC|GMT|CET|CEST|MEZ|MESZ|PST|PDT|EST|EDT)\b\s*[+\-]?\d{0,2}(?::\d{2})?/gi, " ")
|
||||
// Zahlen/Uhrzeiten (18:01 -> „achtzehn Uhr eins", 2026 -> „zweitausend…") normalisiert jetzt
|
||||
// der lokale pocket_server (text_norm.py, num2words) — kontextsicher (Modellnummern bleiben).
|
||||
.replace(/https?:\/\/\S+/gi, " ").replace(/www\.\S+/gi, " ").replace(/\b\S+@\S+\.\S+\b/g, " ")
|
||||
.replace(/[*_#>~|`]+/g, " ").replace(/^\s*[-•·]\s+/gm, " ")
|
||||
.replace(/\s*&\s*/g, " und ")
|
||||
.replace(/(\d)\s*%/g, "$1 Prozent").replace(/%/g, " Prozent ")
|
||||
.replace(/(\d)\s*°\s*C?/g, "$1 Grad").replace(/°/g, " Grad ")
|
||||
.replace(/\s*=\s*/g, " gleich ").replace(/\s*\/\s*/g, " ")
|
||||
.replace(/[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}]/gu, "")
|
||||
.replace(/\s+/g, " ").trim()
|
||||
}
|
||||
|
||||
// FRISCHE Session pro App-Start (NICHT in localStorage persistieren). Sonst wächst der Hermes-Verlauf
|
||||
// über alle Starts hinweg unbegrenzt (gesehen: 97k+ Tokens) -> Degeneration/Wiederhol-Schleifen.
|
||||
// Dauerhaftes Gedächtnis liegt ohnehin in Mem0 (sessionunabhängig) -> Continuity bleibt erhalten.
|
||||
export function newSessionId(): string {
|
||||
return "lucy-" + Math.random().toString(36).slice(2) + Date.now().toString(36)
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Tanzen zur Musik (MateEngine-Idee): nimmt den SYSTEM-Ton per Loopback ab (getDisplayMedia, der
|
||||
// Main-Prozess liefert audio:"loopback") und misst die Bass-Energie. Daraus speist sich danceLevel,
|
||||
// das der Avatar in eine rhythmische Tanzbewegung umsetzt. Kein Video wird verwendet (Track sofort gestoppt).
|
||||
export function useDanceAudio() {
|
||||
const level = useRef(0)
|
||||
const [dancing, setDancing] = useState(false)
|
||||
const ctxRef = useRef<AudioContext | null>(null)
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
const rafRef = useRef(0)
|
||||
|
||||
const stop = useCallback(() => {
|
||||
cancelAnimationFrame(rafRef.current)
|
||||
streamRef.current?.getTracks().forEach((t) => { try { t.stop() } catch { /* */ } })
|
||||
streamRef.current = null
|
||||
ctxRef.current?.close().catch(() => {}); ctxRef.current = null
|
||||
level.current = 0; setDancing(false)
|
||||
}, [])
|
||||
|
||||
const start = useCallback(async () => {
|
||||
try {
|
||||
// Video wird vom Handler verlangt, brauchen wir aber nicht -> sofort stoppen, nur Audio behalten
|
||||
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: true })
|
||||
stream.getVideoTracks().forEach((t) => t.stop())
|
||||
if (stream.getAudioTracks().length === 0) { stream.getTracks().forEach((t) => t.stop()); throw new Error("kein System-Audio") }
|
||||
const ctx = new (window.AudioContext || (window as any).webkitAudioContext)()
|
||||
const src = ctx.createMediaStreamSource(stream)
|
||||
const an = ctx.createAnalyser(); an.fftSize = 256; an.smoothingTimeConstant = 0.7
|
||||
src.connect(an)
|
||||
const freq = new Uint8Array(an.frequencyBinCount)
|
||||
const tick = () => {
|
||||
an.getByteFrequencyData(freq)
|
||||
let s = 0; for (let i = 1; i < 10; i++) s += freq[i] // Bass-/Kick-Bereich
|
||||
level.current = Math.min(1, (s / 9 / 255) * 1.7)
|
||||
rafRef.current = requestAnimationFrame(tick)
|
||||
}
|
||||
tick()
|
||||
ctxRef.current = ctx; streamRef.current = stream
|
||||
setDancing(true)
|
||||
} catch (e) { console.error("Tanz-Audio:", e); stop() }
|
||||
}, [stop])
|
||||
|
||||
const toggleDance = useCallback(() => { dancing ? stop() : void start() }, [dancing, start, stop])
|
||||
|
||||
useEffect(() => () => stop(), [stop])
|
||||
return { danceLevel: level, dancing, toggleDance }
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
// Push-to-talk-Aufnahme via MediaRecorder. start beim Druecken, stop beim Loslassen -> Audio-Blob an onAudio.
|
||||
export function usePushToTalk(onAudio: (blob: Blob) => void) {
|
||||
const [recording, setRecording] = useState(false)
|
||||
const recRef = useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = useRef<Blob[]>([])
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (recRef.current) return
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
console.error("Mikrofon-Zugriff verweigert:", e)
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/webm"
|
||||
const rec = new MediaRecorder(stream, { mimeType: mime })
|
||||
chunksRef.current = []
|
||||
rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) }
|
||||
rec.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: mime })
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current = null
|
||||
recRef.current = null
|
||||
setRecording(false)
|
||||
if (blob.size > 1200) onAudio(blob)
|
||||
}
|
||||
rec.start()
|
||||
recRef.current = rec
|
||||
setRecording(true)
|
||||
}, [onAudio])
|
||||
|
||||
const stop = useCallback(() => { recRef.current?.stop() }, [])
|
||||
|
||||
useEffect(() => () => {
|
||||
recRef.current?.stop()
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
}, [])
|
||||
|
||||
return { recording, start, stop }
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
import { useEffect, useRef } from "react"
|
||||
import { MicVAD } from "@ricky0123/vad-web"
|
||||
import { BOX_URL } from "../../config"
|
||||
|
||||
// Freisprech-VAD: lauscht dauerhaft am Mikro und liefert komplette Äußerungen als Blob.
|
||||
// Seit Review P1-7 (2026-07-02) Silero VAD v5 (neuronal, via vad-web/onnxruntime-wasm) statt
|
||||
// des RMS-Eigenbaus: erkennt Sprache statt Lautstärke (robust gegen Tastatur/Lüfter) und
|
||||
// erlaubt dadurch ein kürzeres Turn-Ende (450 ms statt 900 ms Stille) ohne Abschneiden.
|
||||
// RÜCKKOPPLUNGS-SCHUTZ bleibt: `paused` (Lucy denkt/spricht) verwirft laufende Erkennung,
|
||||
// + echoCancellation + Abkling-Sperre nach Lucys Antwort.
|
||||
|
||||
interface VADOptions {
|
||||
enabled: boolean // VAD-Modus an?
|
||||
paused: boolean // Lucy beschäftigt (thinking/speaking/listening) -> nicht aufnehmen
|
||||
onUtterance: (blob: Blob) => void
|
||||
onListening?: (active: boolean) => void // UI: gerade Sprache am Aufnehmen?
|
||||
}
|
||||
|
||||
const REDEMPTION_MS = 450 // so lange Stille -> Äußerung zu Ende (war 900 ms beim RMS-VAD)
|
||||
const PRE_SPEECH_PAD_MS = 320 // Vorlauf mitschneiden (erster Wortanfang nicht abschneiden)
|
||||
const MIN_SPEECH_MS = 160 // kürzer = Klick/Räuspern -> verwerfen
|
||||
const COOLDOWN_MS = 450 // nach Lucys Antwort kurz taub (Echo/Lautsprecher abklingen lassen)
|
||||
// Semantische Turn-Detection (Smart Turn v3 auf der Box): meldet sie 'unfertig' (User denkt
|
||||
// mitten im Satz nach), warten wir bis zu HOLD_MS auf die Fortsetzung und hängen sie an,
|
||||
// statt mitten im Gedanken zu antworten. Hart begrenzt, damit Lucy nie ewig schweigt.
|
||||
// KONSERVATIV seit 02.07. abends: Nur bei SEHR sicherem 'unfertig' warten (P(fertig) < 0.15) —
|
||||
// beim ersten Realtest hielt der Check zu viele echte Sätze auf (+1,8s je Turn, „sehr langsam").
|
||||
// Not-Aus ohne Rebuild: localStorage lucy_turncheck = "0".
|
||||
const HOLD_MS = 1200
|
||||
const HOLD_BELOW_P = 0.15 // nur unter dieser P(fertig) wird gewartet
|
||||
const MAX_UTTERANCE_S = 30 // Sicherheitsdeckel fürs Zusammenhängen
|
||||
const TURNCHECK_ENABLED = typeof localStorage === "undefined" || localStorage.getItem("lucy_turncheck") !== "0"
|
||||
|
||||
// Float32-Samples (16 kHz mono) -> WAV-Blob (PCM16). Ersetzt den MediaRecorder-webm-Umweg:
|
||||
// die Box muss kein Opus mehr dekodieren, Parakeet/Whisper bekommen direkt sauberes WAV.
|
||||
function toWavBlob(samples: Float32Array, sampleRate = 16000): Blob {
|
||||
const buf = new ArrayBuffer(44 + samples.length * 2)
|
||||
const v = new DataView(buf)
|
||||
const writeStr = (off: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(off + i, s.charCodeAt(i)) }
|
||||
writeStr(0, "RIFF"); v.setUint32(4, 36 + samples.length * 2, true); writeStr(8, "WAVE")
|
||||
writeStr(12, "fmt "); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true)
|
||||
v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true)
|
||||
writeStr(36, "data"); v.setUint32(40, samples.length * 2, true)
|
||||
let off = 44
|
||||
for (let i = 0; i < samples.length; i++, off += 2) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]))
|
||||
v.setInt16(off, s < 0 ? s * 0x8000 : s * 0x7fff, true)
|
||||
}
|
||||
return new Blob([buf], { type: "audio/wav" })
|
||||
}
|
||||
|
||||
// Fragt die Box nach P(fertig) der Äußerung. Fehler => 1.0 (nie blockieren, sofort senden).
|
||||
async function turnCompleteness(audio: Float32Array): Promise<number> {
|
||||
try {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", toWavBlob(audio.subarray(Math.max(0, audio.length - 8 * 16000))), "rec.wav")
|
||||
const r = await fetch(`${BOX_URL}/api/voice/turn`, { method: "POST", body: fd })
|
||||
if (!r.ok) return 1.0
|
||||
const j = await r.json()
|
||||
return typeof j.probability === "number" ? j.probability : 1.0
|
||||
} catch {
|
||||
return 1.0
|
||||
}
|
||||
}
|
||||
|
||||
function concatAudio(a: Float32Array, b: Float32Array): Float32Array {
|
||||
const out = new Float32Array(a.length + b.length)
|
||||
out.set(a); out.set(b, a.length)
|
||||
return out
|
||||
}
|
||||
|
||||
export function useVAD({ enabled, paused, onUtterance, onListening }: VADOptions) {
|
||||
const pausedRef = useRef(paused); pausedRef.current = paused
|
||||
const onUtt = useRef(onUtterance); onUtt.current = onUtterance
|
||||
const onLst = useRef(onListening); onLst.current = onListening
|
||||
const resumeAt = useRef(0)
|
||||
|
||||
// Abkling-Sperre: sobald Lucy fertig ist (paused true->false), kurz nicht lauschen
|
||||
useEffect(() => { if (!paused) resumeAt.current = performance.now() + COOLDOWN_MS }, [paused])
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) return
|
||||
let cancelled = false
|
||||
let vad: Awaited<ReturnType<typeof MicVAD.new>> | null = null
|
||||
// Semantik-Hold: bei 'incomplete' gepufferte Äußerung, auf die die Fortsetzung wartet.
|
||||
let pending: Float32Array | null = null
|
||||
let holdTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const emit = (audio: Float32Array) => {
|
||||
pending = null
|
||||
if (holdTimer) { clearTimeout(holdTimer); holdTimer = null }
|
||||
onUtt.current(toWavBlob(audio))
|
||||
}
|
||||
|
||||
;(async () => {
|
||||
try {
|
||||
vad = await MicVAD.new({
|
||||
model: "v5",
|
||||
// Assets self-hosted (electron.vite.config.ts kopiert sie nach /vad/) — offlinefähig.
|
||||
baseAssetPath: "/vad/",
|
||||
onnxWASMBasePath: "/vad/",
|
||||
positiveSpeechThreshold: 0.5,
|
||||
negativeSpeechThreshold: 0.35,
|
||||
redemptionMs: REDEMPTION_MS,
|
||||
preSpeechPadMs: PRE_SPEECH_PAD_MS,
|
||||
minSpeechMs: MIN_SPEECH_MS,
|
||||
getStream: () => navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
}),
|
||||
onSpeechStart: () => {
|
||||
if (pausedRef.current || performance.now() < resumeAt.current) return
|
||||
// User spricht weiter, während eine 'incomplete'-Äußerung gehalten wird ->
|
||||
// Timer stoppen; die Fortsetzung wird in onSpeechEnd angehängt.
|
||||
if (holdTimer) { clearTimeout(holdTimer); holdTimer = null }
|
||||
onLst.current?.(true)
|
||||
},
|
||||
onSpeechEnd: async (audio: Float32Array) => {
|
||||
onLst.current?.(false)
|
||||
// Während Lucy denkt/spricht (oder direkt danach) erkannte Sprache = ihr eigenes
|
||||
// Echo bzw. Nachhall -> verwerfen statt transkribieren.
|
||||
if (cancelled || pausedRef.current || performance.now() < resumeAt.current) { pending = null; return }
|
||||
if (!pending && audio.length < MIN_SPEECH_MS * 16) return // 16 Samples/ms @16 kHz
|
||||
let combined = pending ? concatAudio(pending, audio) : audio
|
||||
if (combined.length > MAX_UTTERANCE_S * 16000) {
|
||||
combined = combined.subarray(combined.length - MAX_UTTERANCE_S * 16000)
|
||||
}
|
||||
// Bereits einmal gehalten? Dann JETZT senden — maximal eine Warterunde pro Äußerung.
|
||||
if (!TURNCHECK_ENABLED || pending) { emit(combined); return }
|
||||
const t0 = performance.now()
|
||||
const pComplete = await turnCompleteness(combined)
|
||||
if (cancelled) return
|
||||
if (localStorage.getItem("lucy_perf") === "1") {
|
||||
console.log(`[lucy-perf] Turn-Check P(fertig)=${pComplete} in ${Math.round(performance.now() - t0)}ms -> ${pComplete < HOLD_BELOW_P ? "HALTEN" : "senden"}`)
|
||||
}
|
||||
if (pComplete >= HOLD_BELOW_P) { emit(combined); return }
|
||||
// SEHR sicher mitten im Gedanken: kurz auf die Fortsetzung warten, dann notfalls senden.
|
||||
pending = combined
|
||||
if (holdTimer) clearTimeout(holdTimer)
|
||||
holdTimer = setTimeout(() => { if (!cancelled && pending) emit(pending) }, HOLD_MS)
|
||||
},
|
||||
onVADMisfire: () => onLst.current?.(false),
|
||||
})
|
||||
if (cancelled) { vad.destroy(); return }
|
||||
vad.start()
|
||||
} catch (e) {
|
||||
console.error("VAD: Silero-Init fehlgeschlagen", e)
|
||||
}
|
||||
})()
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
if (holdTimer) clearTimeout(holdTimer)
|
||||
try { vad?.destroy() } catch { /* */ }
|
||||
onLst.current?.(false)
|
||||
}
|
||||
}, [enabled])
|
||||
}
|
||||
@@ -1,305 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { usePushToTalk } from "./usePushToTalk"
|
||||
import { useVAD } from "./useVAD"
|
||||
import { AudioQueue } from "./audio"
|
||||
import { sentimentToEmotion, type Emotion } from "./sentiment"
|
||||
import { BOX_URL, SYSTEM_PROMPT } from "../../config"
|
||||
import { stt, tts, SpeechScheduler } from "../../voice-core"
|
||||
import { EMO_TAG, cleanForTTS, newSessionId, restoreUmlauts, stripEmoTag, stripMedia } from "./textPipeline"
|
||||
import { VISION_RX, extractWindowName } from "./visionIntent"
|
||||
import { PERF, plog, plogSend } from "./perf"
|
||||
|
||||
// Voll-Duplex-Schleife: PTT/VAD -> Box /stt -> Box /chat (SSE Hermes) -> CHUNKS -> lokal /tts -> AudioQueue.
|
||||
// Hirn+STT = Box (wie WebUI), Stimme = lokal (pocket-tts, CPU). Antwort wird in groessere Bloecke
|
||||
// gebuendelt und satzweise frueh gesprochen. Warm-Gate beim Start: Bedienung erst frei, wenn der
|
||||
// TTS-Dienst antwortet. Text-Verarbeitung: textPipeline.ts · Vision-Intent: visionIntent.ts ·
|
||||
// Perf-Logging: perf.ts (Review P2-13a: Hook = nur noch Orchestrierung).
|
||||
|
||||
export type VoiceStatus = "warming" | "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error"
|
||||
export interface ChatMsg { role: "user" | "assistant"; text: string }
|
||||
|
||||
export function useVoiceAgent() {
|
||||
const [status, setStatus] = useState<VoiceStatus>("warming")
|
||||
const [ready, setReady] = useState(false)
|
||||
const [messages, setMessages] = useState<ChatMsg[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [inputMode, setInputModeState] = useState<"ptt" | "vad">(
|
||||
() => (localStorage.getItem("lucy_input_mode") === "vad" ? "vad" : "ptt"))
|
||||
const [vadListening, setVadListening] = useState(false)
|
||||
const setInputMode = useCallback((m: "ptt" | "vad") => {
|
||||
localStorage.setItem("lucy_input_mode", m); setInputModeState(m)
|
||||
}, [])
|
||||
|
||||
const audioLevel = useRef({ current: 0 })
|
||||
const emotion = useRef<Emotion>("neutral")
|
||||
const queueRef = useRef<AudioQueue | null>(null)
|
||||
const schedulerRef = useRef<SpeechScheduler | null>(null)
|
||||
const turnAbortRef = useRef<AbortController | null>(null) // laufenden Turn abbrechen (Barge-in)
|
||||
const thinkingRef = useRef(false) // Hermes werkelt noch (Tools) -> Status nach Filler zurück auf 'thinking'
|
||||
const turnT0 = useRef(0) // Perf: Startzeit des aktuellen Turns (Mikro losgelassen)
|
||||
const sessionId = useRef<string>(newSessionId())
|
||||
// Live-Status fuer die Eingabe-Sperre (ohne pressStart-Closure neu zu binden)
|
||||
const statusRef = useRef<VoiceStatus>("warming")
|
||||
useEffect(() => { statusRef.current = status }, [status])
|
||||
|
||||
const ensureQueue = useCallback(() => {
|
||||
if (!queueRef.current) {
|
||||
const q = new AudioQueue()
|
||||
queueRef.current = q
|
||||
audioLevel.current = q.level
|
||||
// Satz-Pipelining: der Scheduler spricht einzelne Sätze, sobald sie aus dem Hirn-Stream
|
||||
// fertig sind. Er (nicht die Queue) treibt den „speaking"-Status, damit es zwischen Sätzen
|
||||
// nicht flackert (die Queue pausiert die Pegelmessung je Satz).
|
||||
const sch = new SpeechScheduler(tts, q)
|
||||
// Nach dem Filler-Satz zurück auf 'thinking', solange Hermes noch werkelt (sonst flackert's auf idle).
|
||||
sch.onBusy = (busy) => setStatus((s) => (busy ? "speaking" : thinkingRef.current ? "thinking" : s === "speaking" ? "idle" : s))
|
||||
schedulerRef.current = sch
|
||||
}
|
||||
return queueRef.current
|
||||
}, [])
|
||||
|
||||
// --- Warm-Gate: warten bis der lokale TTS-Dienst geladen ist, bevor die Bedienung frei wird ---
|
||||
// pocket_server serviert /health erst, wenn das Modell fertig geladen ist (FastAPI-lifespan).
|
||||
// Der Main-Prozess spawnt den Server beim App-Start; hier nur pollen.
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
;(async () => {
|
||||
// Geduldiger Retry mit Backoff statt einmaligem 120s-Warten: crasht der pocket_server
|
||||
// (oder braucht er länger), bleibt die App in 'warming' MIT sichtbarer Meldung — vorher
|
||||
// schaltete sie nach dem Timeout stumm auf 'ready' und die Stimme fehlte einfach.
|
||||
let attempt = 0
|
||||
while (!cancelled) {
|
||||
const ok = await tts.waitReady(attempt === 0 ? 45_000 : 20_000)
|
||||
if (cancelled) return
|
||||
if (ok) break
|
||||
attempt++
|
||||
setError(`Stimme startet nicht (Versuch ${attempt}) — pocket_server prüfen, ich versuche es weiter …`)
|
||||
await new Promise((r) => setTimeout(r, Math.min(5_000 * attempt, 30_000)))
|
||||
}
|
||||
if (cancelled) return
|
||||
setError(null)
|
||||
// ein Aufwaerm-Satz (primt alle lazy Pfade); Audio verwerfen
|
||||
try { await tts.synthesize("Alles bereit, Commander.") } catch { /* */ }
|
||||
if (cancelled) return
|
||||
setReady(true); setStatus("idle")
|
||||
})()
|
||||
return () => { cancelled = true }
|
||||
}, [])
|
||||
|
||||
// Ein kompletter Turn: User-Text (+ optional Bildschirm-Bilder, Multi-Monitor) -> Hermes (SSE) -> Stimme.
|
||||
const runTurn = useCallback(async (userText: string, images?: string[]) => {
|
||||
const queue = ensureQueue()
|
||||
const scheduler = schedulerRef.current!
|
||||
// Vorherigen Turn (falls noch am Streamen) hart abbrechen -> kein „Weiterreden" nach Barge-in.
|
||||
turnAbortRef.current?.abort()
|
||||
const ac = new AbortController()
|
||||
turnAbortRef.current = ac
|
||||
scheduler.clear(); queue.clear() // frischer Turn: evtl. Reste aus vorherigem Sprechen verwerfen
|
||||
setStatus("thinking")
|
||||
thinkingRef.current = true // Hermes werkelt (evtl. Tools) -> Status nach Filler zurück auf 'thinking'
|
||||
let assistant = ""
|
||||
let firstToken = true
|
||||
setMessages((m) => [...m, { role: "assistant", text: "" }])
|
||||
|
||||
// GANZE Antwort in EINEM Stream an Pocket (nach Hermes-Ende). Pocket kürzt intern den ersten
|
||||
// Chunk (FAST_FIRST) -> schnelles erstes Audio, konsistente Prosodie, wenige Collapse-Regens.
|
||||
// (Client-seitiges Satz-Chunking kämpfte gegen genau diese Optimierung -> verworfen.)
|
||||
let dispatchedAny = false
|
||||
let aborted = false
|
||||
// --- Inkrementelles Sprechen bei TOOL-Turns ------------------------------------------------
|
||||
// Ohne Tools bleibt alles Ein-Stück (Flush erst am Ende) -> gleiche Prosodie/FAST_FIRST wie bisher.
|
||||
// Sobald Hermes ein Tool anstößt (hermes.*-Event), sprechen wir die bis dahin FERTIGEN Sätze schon
|
||||
// -> Lucy redet, WÄHREND das Tool läuft, statt am Ende alles am Stück (sonst 30s+ Totstille bei Tool-Ketten).
|
||||
let spokenLen = 0
|
||||
const SENT_END = /[.!?…](?=[\s"“”„)\]]|$)/g
|
||||
const flushSpeakable = (force: boolean) => {
|
||||
const raw = assistant.slice(spokenLen)
|
||||
if (!raw.trim()) return
|
||||
if (!force && ((raw.match(/```/g)?.length || 0) % 2) === 1) return // offener Code-Zaun -> warten
|
||||
let upto = raw.length
|
||||
if (!force) {
|
||||
let last = -1, m: RegExpExecArray | null
|
||||
SENT_END.lastIndex = 0
|
||||
while ((m = SENT_END.exec(raw))) last = m.index + 1
|
||||
if (last < 0) return // noch kein ganzer Satz fertig -> warten
|
||||
upto = last
|
||||
}
|
||||
spokenLen += upto
|
||||
const seg = cleanForTTS(stripEmoTag(raw.slice(0, upto)))
|
||||
if (seg) { scheduler.push(seg); dispatchedAny = true }
|
||||
}
|
||||
// Sofort-Kontext: lokale Zeit/Datum mitgeben -> Lucy braucht dafür KEIN Tool (spart den 13s-Tool-Tanz).
|
||||
const timeCtx = `\n\n[Sofort-Kontext, DIREKT nutzbar OHNE Tool: Lokale Zeit/Datum beim Commander ist ` +
|
||||
`${new Date().toLocaleString("de-DE", { weekday: "long", day: "numeric", month: "long", year: "numeric", hour: "2-digit", minute: "2-digit" })} Uhr.]`
|
||||
try {
|
||||
const r = await fetch(`${BOX_URL}/api/voice/chat`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text: userText, session_id: sessionId.current, system: SYSTEM_PROMPT + timeCtx, images: images || [] }),
|
||||
signal: ac.signal,
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`)
|
||||
const reader = r.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let sse = ""
|
||||
for (;;) {
|
||||
if (ac.signal.aborted) break
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
sse += dec.decode(value, { stream: true })
|
||||
const events = sse.split("\n\n"); sse = events.pop() || ""
|
||||
for (const ev of events) {
|
||||
const lines = ev.split("\n")
|
||||
// Hermes mischt Tool-Fortschritt (event: hermes.tool.progress) in den Stream. Das sind KEINE
|
||||
// Chat-Chunks -> nicht als solche parsen (ein Tool-Hickup mit error-Objekt darf die Antwort
|
||||
// nicht abbrechen). Optional koennte man hier Tool-Status anzeigen.
|
||||
const evType = lines.find((l) => l.startsWith("event:"))?.slice(6).trim()
|
||||
if (evType && evType.startsWith("hermes.")) {
|
||||
// Tool-Fortschritt (Status bleibt 'thinking'). Für die „denkt-lange"-Diagnose zeigen wir
|
||||
// die Agent-Aktivität mit Zeitstempel -> so sieht man, ob Tools/Reasoning die Zeit fressen.
|
||||
if (PERF) {
|
||||
const dt = lines.find((l) => l.startsWith("data:"))?.slice(5).trim().slice(0, 140) || ""
|
||||
plogSend(`Hermes ${evType} +${Math.round(performance.now() - turnT0.current)}ms ${dt}`)
|
||||
}
|
||||
flushSpeakable(false) // Tool läuft an -> die bis hier fertigen Sätze schon sprechen (redet WÄHREND das Tool arbeitet)
|
||||
continue
|
||||
}
|
||||
const line = lines.find((l) => l.startsWith("data:"))
|
||||
if (!line) continue
|
||||
const data = line.slice(5).trim()
|
||||
if (data === "[DONE]") continue
|
||||
let json: any
|
||||
try { json = JSON.parse(data) } catch { continue }
|
||||
if (json.error) {
|
||||
const msg = typeof json.error === "string"
|
||||
? json.error
|
||||
: (json.error?.message || JSON.stringify(json.error))
|
||||
throw new Error(msg)
|
||||
}
|
||||
const delta = json.choices?.[0]?.delta?.content || ""
|
||||
if (!delta) continue
|
||||
if (firstToken) { firstToken = false; plog("Chat-TTFB (1. Hermes-Token)", performance.now() - turnT0.current) }
|
||||
assistant += delta
|
||||
// Defensiv: falls Hermes seinen internen Umschlag durchreicht (Mid-Turn) -> NICHTS vorlesen.
|
||||
if (assistant.includes("OUT-OF-BAND USER MESSAGE")) { aborted = true; scheduler.clear(); queue.clear(); break }
|
||||
emotion.current = sentimentToEmotion(assistant)
|
||||
setMessages((m) => { const c = m.slice(); c[c.length - 1] = { role: "assistant", text: stripEmoTag(stripMedia(restoreUmlauts(assistant))) }; return c })
|
||||
}
|
||||
if (aborted) break
|
||||
}
|
||||
if (ac.signal.aborted) return // Turn wurde unterbrochen (Barge-in) -> still beenden
|
||||
if (aborted) {
|
||||
setStatus("error")
|
||||
setError("Nachricht kam mitten im Turn an — bitte warten, bis Lucy fertig ist, dann erneut fragen.")
|
||||
return
|
||||
}
|
||||
if (!assistant.trim()) { setStatus("idle"); return }
|
||||
// Stimmungs-Tag (falls vorhanden) -> Avatar-Mimik nach BEDEUTUNG (sonst bleibt Sentiment-Heuristik).
|
||||
const emo = assistant.match(EMO_TAG)
|
||||
if (emo) emotion.current = emo[1].toLowerCase() as Emotion
|
||||
// Rest sprechen: bei Tool-Turns wurde schon segmentweise geflusht; hier kommt das letzte Segment
|
||||
// (bzw. bei Turns OHNE Tools die GANZE Antwort in einem Rutsch -> FAST_FIRST, gleiche Prosodie wie bisher).
|
||||
thinkingRef.current = false // Hermes fertig -> nach der Antwort darf der Status auf idle (kein Rückfall auf 'thinking')
|
||||
plog("Text an TTS (Rest)", performance.now() - turnT0.current)
|
||||
flushSpeakable(true)
|
||||
await scheduler.idle() // warten, bis Lucy fertig gesprochen hat
|
||||
if (dispatchedAny && !scheduler.playedAny) {
|
||||
setStatus("error")
|
||||
setError("Sprachausgabe fehlgeschlagen — laeuft der lokale TTS-Dienst?")
|
||||
} else if (!dispatchedAny) {
|
||||
setStatus("idle")
|
||||
}
|
||||
} catch (e: any) {
|
||||
if (e?.name === "AbortError" || ac.signal.aborted) return // absichtlich abgebrochen (Barge-in)
|
||||
setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`)
|
||||
}
|
||||
}, [ensureQueue])
|
||||
|
||||
// Bildschirm-Sicht: Bild(er) erfassen (gezieltes Fenster per Name, sonst ALLE Monitore)
|
||||
const captureForVision = useCallback(async (userText: string): Promise<{ images: string[]; label: string }> => {
|
||||
if (!window.lucy) return { images: [], label: "" }
|
||||
try {
|
||||
const name = extractWindowName(userText)
|
||||
if (name) {
|
||||
const hit = await window.lucy.captureByName(name)
|
||||
if (hit?.image) return { images: [hit.image], label: ` 👁 ${hit.name}` }
|
||||
}
|
||||
const shots = (await window.lucy.captureScreen()).filter(Boolean)
|
||||
if (shots.length) return { images: shots, label: shots.length > 1 ? ` 👁 ${shots.length} Bildschirme` : " 👁 Bildschirm" }
|
||||
} catch (e) { console.error("Bildschirm-Capture:", e) }
|
||||
return { images: [], label: "" }
|
||||
}, [])
|
||||
|
||||
const handleAudio = useCallback(async (blob: Blob) => {
|
||||
setError(null)
|
||||
turnT0.current = performance.now() // Perf: Startpunkt = Mikro losgelassen
|
||||
turnAbortRef.current?.abort() // Barge-in: laufenden Hirn-Turn stoppen (kein Nachschieben)
|
||||
thinkingRef.current = false
|
||||
ensureQueue().clear()
|
||||
schedulerRef.current?.clear() // Barge-in: wartende Sätze mit verwerfen
|
||||
setStatus("transcribing")
|
||||
let userText = ""
|
||||
try {
|
||||
userText = await stt.transcribe(blob)
|
||||
plog("STT (Mikro->Text)", performance.now() - turnT0.current)
|
||||
} catch (e: any) {
|
||||
setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return
|
||||
}
|
||||
if (!userText) { setStatus("idle"); return }
|
||||
// Bittet der Nutzer Lucy, auf den Bildschirm zu schauen? -> Bild(er) erfassen + mitschicken.
|
||||
const vis = VISION_RX.test(userText) ? await captureForVision(userText) : { images: [] as string[], label: "" }
|
||||
setMessages((m) => [...m, { role: "user", text: userText + vis.label }])
|
||||
await runTurn(userText, vis.images)
|
||||
}, [ensureQueue, runTurn, captureForVision])
|
||||
|
||||
// Auge-Button: Lucy aktiv auf den Bildschirm schauen lassen (ohne Sprachbefehl)
|
||||
const lookAtScreen = useCallback(async () => {
|
||||
if (!ready || statusRef.current === "thinking" || statusRef.current === "transcribing") return
|
||||
setError(null); turnT0.current = performance.now(); ensureQueue().clear(); schedulerRef.current?.clear()
|
||||
let images: string[] = []
|
||||
try { images = ((await window.lucy?.captureScreen()) || []).filter(Boolean) } catch { /* */ }
|
||||
const prompt = "Schau auf meinen Bildschirm und sag mir kurz, was du darauf siehst."
|
||||
const label = images.length > 1 ? ` 👁 ${images.length} Bildschirme` : " 👁 Bildschirm"
|
||||
setMessages((m) => [...m, { role: "user", text: prompt + label }])
|
||||
await runTurn(prompt, images)
|
||||
}, [ensureQueue, runTurn, ready])
|
||||
|
||||
const { recording, start, stop } = usePushToTalk(handleAudio)
|
||||
|
||||
// Freisprech-VAD: nur im vad-Modus, und nur wenn Lucy IDLE ist (paused sonst) -> keine Rückkopplung,
|
||||
// keine Mid-Turn-Nachricht. ensureQueue() vor handleAudio, damit Barge-in/Audio bereit ist.
|
||||
useVAD({
|
||||
enabled: ready && inputMode === "vad",
|
||||
paused: status !== "idle",
|
||||
onUtterance: (blob) => { ensureQueue(); void handleAudio(blob) },
|
||||
onListening: setVadListening,
|
||||
})
|
||||
|
||||
const pressStart = useCallback(() => {
|
||||
if (!ready) return
|
||||
// Waehrend das Gehirn arbeitet (STT/Hermes-Turn laeuft) KEINEN neuen Turn starten: eine zweite
|
||||
// Nachricht mitten im Turn liefert Hermes als "OUT-OF-BAND" aus -> Modell echot den Umschlag.
|
||||
// Barge-in beim SPRECHEN bleibt erlaubt (Hermes-Turn ist dann fertig) -> queue.clear() in handleAudio.
|
||||
if (statusRef.current === "transcribing" || statusRef.current === "thinking") return
|
||||
ensureQueue()
|
||||
setStatus("listening")
|
||||
void start()
|
||||
}, [ready, ensureQueue, start])
|
||||
|
||||
const pressEnd = useCallback(() => { stop() }, [stop])
|
||||
|
||||
const reset = useCallback(() => {
|
||||
turnAbortRef.current?.abort()
|
||||
thinkingRef.current = false
|
||||
queueRef.current?.clear()
|
||||
schedulerRef.current?.clear()
|
||||
setMessages([]); setError(null); setStatus("idle")
|
||||
sessionId.current = newSessionId() // frischer Gesprächsfaden (Mem0-Gedächtnis bleibt)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!recording && status === "listening") setStatus("transcribing")
|
||||
}, [recording, status])
|
||||
|
||||
return { status, ready, messages, error, recording, audioLevel, emotion, pressStart, pressEnd, reset,
|
||||
inputMode, setInputMode, vadListening, lookAtScreen }
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
// Bildschirm-Sicht-Intent: erkennt im Gesagten die Bitte, auf den Schirm zu schauen
|
||||
// (aus useVoiceAgent extrahiert, Review P2-13a).
|
||||
|
||||
export const VISION_RX = /\b(schau|sieh|siehst|guck|guckst|zeig|bildschirm|screen|monitor|fenster|erkennst?|lies (mir|das)|was (steht|ist) (da|hier|auf)|auf meinem (bildschirm|schirm|screen))\b/i
|
||||
|
||||
export function extractWindowName(text: string): string | null {
|
||||
// grobe Heuristik: Wort/Phrase nach 'fenster|app|programm|in|auf|bei|im'
|
||||
const m = text.match(/(?:fenster|app|programm|in|auf|bei|im)\s+([A-Za-zÄÖÜäöü][\w.+\- ]{2,28})/i)
|
||||
if (!m) return null
|
||||
return m[1].replace(/\b(an|fenster|programm|app|bildschirm|schirm|screen|siehst|du)\b/gi, "").trim() || null
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import React from "react"
|
||||
import ReactDOM from "react-dom/client"
|
||||
import App from "./App"
|
||||
import "./styles.css"
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
)
|
||||
@@ -1,282 +0,0 @@
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
:root { color-scheme: dark; }
|
||||
html, body, #root { height: 100%; }
|
||||
body {
|
||||
font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
|
||||
color: #e5e7eb; overflow: hidden; background: transparent; /* frameless+transparent: App malt den Hintergrund */
|
||||
}
|
||||
|
||||
/* ---- App-Rahmen: Titelleiste + Body ---- */
|
||||
.app { display: flex; flex-direction: column; height: 100%; overflow: hidden; }
|
||||
.app.full {
|
||||
background: radial-gradient(1200px 800px at 50% 30%, #14141f 0%, #0a0a0f 70%);
|
||||
border: 1px solid rgba(255,255,255,0.08); border-radius: 14px;
|
||||
}
|
||||
.app.overlay { background: transparent; }
|
||||
|
||||
.titlebar {
|
||||
flex-shrink: 0; height: 34px; display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 0 6px 0 12px; -webkit-app-region: drag; user-select: none;
|
||||
}
|
||||
.app.overlay .titlebar { height: 28px; opacity: 0.35; transition: opacity 0.2s; }
|
||||
.app.overlay:hover .titlebar { opacity: 1; }
|
||||
.drag { display: flex; align-items: center; gap: 8px; min-width: 0; flex: 1; }
|
||||
.dot { width: 9px; height: 9px; border-radius: 50%; flex-shrink: 0; box-shadow: 0 0 8px currentColor; }
|
||||
.ttl { font-size: 12px; font-weight: 600; letter-spacing: 0.04em; color: #cbd5e1; }
|
||||
.ttlStatus { font-size: 11px; color: #6b7280; margin-left: 6px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.winbtns { display: flex; gap: 2px; -webkit-app-region: no-drag; }
|
||||
.wb {
|
||||
width: 28px; height: 24px; border: none; background: transparent; color: #9ca3af; cursor: pointer;
|
||||
border-radius: 6px; font-size: 13px; display: flex; align-items: center; justify-content: center;
|
||||
}
|
||||
.wb:hover { background: rgba(255,255,255,0.1); color: #e5e7eb; }
|
||||
.wb.close:hover { background: #ef4444; color: #fff; }
|
||||
.wb[data-on="true"] { color: #7dd3fc; }
|
||||
|
||||
.body { flex: 1; min-height: 0; display: flex; gap: 14px; padding: 0 14px 14px; }
|
||||
.app.overlay .body { padding: 0; }
|
||||
|
||||
.stage {
|
||||
position: relative; display: flex; flex-direction: column; flex: 1; min-width: 0;
|
||||
border: 1px solid rgba(255,255,255,0.07); border-radius: 14px;
|
||||
background: rgba(255,255,255,0.02); overflow: hidden;
|
||||
}
|
||||
.app.overlay .stage { border: none; background: transparent; }
|
||||
.avatarWrap { flex: 1; min-height: 0; position: relative; overflow: hidden; border-radius: 14px; }
|
||||
/* Overlay: Lucy am Koerper greifen & ueber den Desktop ziehen (Bedienelemente bleiben no-drag) */
|
||||
.app.overlay .avatarWrap { -webkit-app-region: drag; }
|
||||
.app.overlay .overlayBubbles, .app.overlay .overlayBubbles * { -webkit-app-region: no-drag; }
|
||||
|
||||
/* --- digitaler Raum (animierter Sci-Fi-Backdrop, Etappe 2B) --- */
|
||||
.digitalRoom {
|
||||
position: absolute; inset: 0; z-index: 0; overflow: hidden;
|
||||
background:
|
||||
radial-gradient(60% 50% at 50% 16%, rgba(125,211,252,0.10), transparent 70%),
|
||||
radial-gradient(70% 60% at 82% 92%, rgba(167,139,250,0.10), transparent 70%),
|
||||
radial-gradient(60% 60% at 12% 88%, rgba(52,211,153,0.06), transparent 70%),
|
||||
linear-gradient(180deg, #0b0b15 0%, #07070d 100%);
|
||||
box-shadow: inset 0 0 130px 36px rgba(0,0,0,0.55); /* Vignette -> Tiefe */
|
||||
}
|
||||
.digitalRoom::before { /* sanft driftendes Aurora-Licht (KEIN filter:blur -> Verlaeufe sind weich genug, spart GPU) */
|
||||
content: ""; position: absolute; inset: -20%; z-index: 0; will-change: transform;
|
||||
background:
|
||||
radial-gradient(40% 40% at 30% 32%, rgba(125,211,252,0.12), transparent 70%),
|
||||
radial-gradient(45% 45% at 74% 42%, rgba(167,139,250,0.12), transparent 70%);
|
||||
animation: auroraDrift 26s ease-in-out infinite alternate;
|
||||
}
|
||||
.digitalRoom::after { /* perspektivisches Neon-Gitter am Boden */
|
||||
content: ""; position: absolute; left: -25%; right: -25%; bottom: -12%; height: 58%;
|
||||
background-image: linear-gradient(rgba(125,211,252,0.13) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(125,211,252,0.10) 1px, transparent 1px);
|
||||
background-size: 46px 46px;
|
||||
transform: perspective(440px) rotateX(62deg); transform-origin: bottom center;
|
||||
-webkit-mask-image: linear-gradient(transparent, #000 65%);
|
||||
animation: gridDrift 14s linear infinite;
|
||||
}
|
||||
/* schwebende Partikel (eine Ebene, nahtlos gekachelt) -> dezente Tiefe, guenstig */
|
||||
.dust {
|
||||
position: absolute; inset: 0; z-index: 0; pointer-events: none; opacity: 0.6;
|
||||
background-image: radial-gradient(1.6px 1.6px at 50% 50%, rgba(125,211,252,0.5), transparent 60%);
|
||||
background-size: 110px 110px; animation: dustRise 30s linear infinite;
|
||||
}
|
||||
@keyframes gridDrift { to { background-position: 0 46px, 46px 0; } }
|
||||
@keyframes auroraDrift { from { transform: translate(-3%, -2%) scale(1); } to { transform: translate(4%, 3%) scale(1.08); } }
|
||||
@keyframes dustRise { to { background-position: 0 -110px; } }
|
||||
.app.overlay .digitalRoom { display: none; } /* Overlay floatet -> nur Avatar + Aura */
|
||||
|
||||
.aura { position: absolute; inset: 0; z-index: 1; pointer-events: none; will-change: transform, opacity; }
|
||||
.avatarCanvas { z-index: 2; }
|
||||
|
||||
.subtitle {
|
||||
position: absolute; left: 7%; right: 7%; bottom: 14px; z-index: 3; text-align: center; max-height: 40%;
|
||||
overflow: hidden; pointer-events: none; border-radius: 10px; padding: 8px 14px;
|
||||
font-size: 15px; line-height: 1.45; color: #eef2f7;
|
||||
text-shadow: 0 1px 6px #000, 0 0 16px rgba(0,0,0,0.6);
|
||||
background: linear-gradient(transparent, rgba(8,8,14,0.45)); animation: subIn 0.2s ease;
|
||||
}
|
||||
@keyframes subIn { from { opacity: 0; transform: translateY(6px); } to { opacity: 1; transform: none; } }
|
||||
.app.overlay .subtitle { font-size: 13px; bottom: 34px; }
|
||||
|
||||
.warmOverlay {
|
||||
position: absolute; inset: 0; display: flex; flex-direction: column;
|
||||
align-items: center; justify-content: center; gap: 14px; text-align: center; padding: 24px;
|
||||
background: rgba(10,10,15,0.72); backdrop-filter: blur(3px); z-index: 5;
|
||||
}
|
||||
.warmTitle { font-size: 16px; color: #e5e7eb; }
|
||||
.warmSub { font-size: 13px; color: #9ca3af; max-width: 340px; line-height: 1.5; }
|
||||
.spinner.big { width: 30px; height: 30px; border-width: 3px; }
|
||||
|
||||
.overlayCaption {
|
||||
position: absolute; left: 0; right: 0; bottom: 8px; display: flex; gap: 6px; justify-content: center;
|
||||
align-items: center; font-size: 12px; color: #cbd5e1; text-shadow: 0 1px 4px #000; pointer-events: none;
|
||||
}
|
||||
|
||||
/* --- Overlay v2: griffbereite Blasen fuer Links/Code (behindern nicht, jederzeit ignorierbar) --- */
|
||||
.overlayBubbles {
|
||||
position: absolute; left: 6px; right: 6px; bottom: 28px; z-index: 4;
|
||||
display: flex; flex-direction: column; gap: 5px; align-items: stretch;
|
||||
}
|
||||
.bubbleChip {
|
||||
display: flex; align-items: center; gap: 4px; padding: 3px 4px 3px 9px; border-radius: 9px;
|
||||
background: rgba(12,12,20,0.82); border: 1px solid rgba(125,211,252,0.25);
|
||||
backdrop-filter: blur(4px); box-shadow: 0 3px 12px rgba(0,0,0,0.4);
|
||||
}
|
||||
.bubbleChip .chipTxt {
|
||||
flex: 1; min-width: 0; font-size: 11px; color: #cbd5e1;
|
||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.bubbleChip button {
|
||||
flex-shrink: 0; width: 22px; height: 20px; border: none; border-radius: 6px; cursor: pointer;
|
||||
background: rgba(125,211,252,0.14); color: #7dd3fc; font-size: 11px; -webkit-app-region: no-drag;
|
||||
}
|
||||
.bubbleChip button:hover { background: rgba(125,211,252,0.28); }
|
||||
|
||||
.controls {
|
||||
flex-shrink: 0; display: flex; flex-direction: column; align-items: center; gap: 12px;
|
||||
padding: 16px; border-top: 1px solid rgba(255,255,255,0.07); background: rgba(0,0,0,0.25);
|
||||
}
|
||||
.app.overlay .controls { border: none; background: transparent; padding: 8px; gap: 6px; }
|
||||
.status { display: flex; align-items: center; gap: 8px; font-size: 14px; }
|
||||
.spinner {
|
||||
width: 14px; height: 14px; border: 2px solid rgba(255,255,255,0.25); border-top-color: #7dd3fc;
|
||||
border-radius: 50%; animation: spin 0.8s linear infinite;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.mic {
|
||||
width: 72px; height: 72px; border-radius: 50%; cursor: pointer; user-select: none;
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
border: 2px solid rgba(125,211,252,0.5); background: rgba(125,211,252,0.12); color: #7dd3fc;
|
||||
transition: transform 0.12s, background 0.12s, border-color 0.12s;
|
||||
}
|
||||
.app.overlay .mic { width: 56px; height: 56px; }
|
||||
.mic:hover { background: rgba(125,211,252,0.2); transform: scale(1.05); }
|
||||
.mic.rec {
|
||||
border-color: #f87171; background: rgba(248,113,113,0.2); color: #fca5a5; transform: scale(1.1);
|
||||
box-shadow: 0 0 24px rgba(248,113,113,0.35);
|
||||
}
|
||||
.mic.disabled { opacity: 0.4; cursor: default; border-color: rgba(255,255,255,0.2); background: rgba(255,255,255,0.04); color: #6b7280; }
|
||||
.mic.disabled:hover { transform: none; background: rgba(255,255,255,0.04); }
|
||||
.hint { font-size: 11px; color: #6b7280; display: flex; align-items: center; gap: 6px; flex-wrap: wrap; justify-content: center; }
|
||||
.hint kbd { background: rgba(255,255,255,0.1); border-radius: 4px; padding: 1px 5px; font-family: monospace; }
|
||||
|
||||
.modeToggle { display: inline-flex; gap: 2px; padding: 2px; border-radius: 9px; background: rgba(255,255,255,0.05); }
|
||||
.modeToggle button {
|
||||
border: none; background: transparent; color: #9ca3af; cursor: pointer;
|
||||
font-size: 11px; padding: 4px 10px; border-radius: 7px; transition: background 0.12s, color 0.12s;
|
||||
}
|
||||
.modeToggle button:hover { color: #e5e7eb; }
|
||||
.modeToggle button.on { background: rgba(125,211,252,0.18); color: #7dd3fc; }
|
||||
.app.overlay .modeToggle { display: none; }
|
||||
.vadDot { width: 7px; height: 7px; border-radius: 50%; background: #34d399; box-shadow: 0 0 8px #34d399; animation: pulse 1s infinite; }
|
||||
|
||||
.danceBtn {
|
||||
border: 1px solid rgba(167,139,250,0.35); background: rgba(167,139,250,0.12); color: #c4b5fd;
|
||||
font-size: 11px; padding: 4px 12px; border-radius: 9px; cursor: pointer; -webkit-app-region: no-drag;
|
||||
transition: background 0.12s, color 0.12s;
|
||||
}
|
||||
.danceBtn:hover { background: rgba(167,139,250,0.22); color: #ddd6fe; }
|
||||
.danceBtn.on { background: rgba(167,139,250,0.3); color: #ede9fe; border-color: rgba(167,139,250,0.6); }
|
||||
.app.overlay .danceBtn { display: none; } /* im Overlay kein Platz -> nur im Vollfenster */
|
||||
|
||||
.side {
|
||||
width: 320px; flex-shrink: 0; display: flex; flex-direction: column;
|
||||
border: 1px solid rgba(255,255,255,0.07); border-radius: 14px; background: rgba(255,255,255,0.02);
|
||||
}
|
||||
.sideHead {
|
||||
display: flex; justify-content: space-between; align-items: center; padding: 12px 16px;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.07);
|
||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.05em; color: #9ca3af;
|
||||
}
|
||||
.iconBtn { background: none; border: none; color: #9ca3af; cursor: pointer; font-size: 15px; }
|
||||
.iconBtn:hover { color: #e5e7eb; }
|
||||
|
||||
.conv { flex: 1; overflow-y: auto; padding: 16px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.empty { font-size: 12px; color: #9ca3af; line-height: 1.6; }
|
||||
.msg { display: flex; flex-direction: column; gap: 4px; }
|
||||
.msg.user { align-items: flex-end; }
|
||||
.msg.assistant { align-items: flex-start; }
|
||||
.who { font-size: 10px; text-transform: uppercase; letter-spacing: 0.05em; color: #6b7280; padding: 0 4px; }
|
||||
.bubble {
|
||||
max-width: 88%; white-space: pre-wrap; word-break: break-word; border-radius: 16px;
|
||||
padding: 8px 12px; font-size: 14px; line-height: 1.5;
|
||||
}
|
||||
.msg.user .bubble { background: rgba(125,211,252,0.14); border-bottom-right-radius: 4px; }
|
||||
.msg.assistant .bubble { background: rgba(255,255,255,0.04); border: 1px solid rgba(255,255,255,0.07); border-bottom-left-radius: 4px; }
|
||||
.dots { display: inline-flex; gap: 4px; }
|
||||
.dots i { width: 6px; height: 6px; border-radius: 50%; background: #6b7280; animation: pulse 1s infinite; }
|
||||
.dots i:nth-child(2) { animation-delay: 0.15s; }
|
||||
.dots i:nth-child(3) { animation-delay: 0.3s; }
|
||||
@keyframes pulse { 0%, 100% { opacity: 0.3; } 50% { opacity: 1; } }
|
||||
|
||||
::-webkit-scrollbar { width: 8px; }
|
||||
::-webkit-scrollbar-thumb { background: rgba(255,255,255,0.12); border-radius: 4px; }
|
||||
|
||||
/* ---- Etappe 2: guter Chat (Markdown, Code-Bloecke, Links, Kopieren) ---- */
|
||||
.bubble { position: relative; }
|
||||
.md { white-space: normal; }
|
||||
.md > :first-child { margin-top: 0; }
|
||||
.md > :last-child { margin-bottom: 0; }
|
||||
.md p { margin: 0 0 8px; }
|
||||
.md ul, .md ol { margin: 4px 0 8px; padding-left: 20px; }
|
||||
.md li { margin: 2px 0; }
|
||||
.md strong { color: #f1f5f9; font-weight: 650; }
|
||||
.md a.mdLink { color: #7dd3fc; text-decoration: underline; cursor: pointer; word-break: break-all; }
|
||||
.md a.mdLink:hover { color: #bae6fd; }
|
||||
.mdInlineCode {
|
||||
font-family: ui-monospace, "Cascadia Code", Consolas, monospace; font-size: 12.5px;
|
||||
background: rgba(125,211,252,0.10); border: 1px solid rgba(125,211,252,0.18);
|
||||
border-radius: 5px; padding: 1px 5px; word-break: break-word;
|
||||
}
|
||||
.codeBlock {
|
||||
margin: 8px 0; border-radius: 10px; overflow: hidden;
|
||||
border: 1px solid rgba(255,255,255,0.10); background: #0c0c14;
|
||||
}
|
||||
.codeBar {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 4px 8px 4px 12px; background: rgba(255,255,255,0.04); border-bottom: 1px solid rgba(255,255,255,0.08);
|
||||
}
|
||||
.codeLang { font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: #6b7280; }
|
||||
.codeBlock pre { margin: 0; padding: 10px 12px; overflow-x: auto; }
|
||||
.codeBlock code {
|
||||
font-family: ui-monospace, "Cascadia Code", Consolas, monospace; font-size: 12.5px; line-height: 1.5;
|
||||
color: #e5e7eb; white-space: pre;
|
||||
}
|
||||
.copyBtn {
|
||||
border: 1px solid rgba(125,211,252,0.3); background: rgba(125,211,252,0.10); color: #7dd3fc;
|
||||
font-size: 11px; padding: 2px 8px; border-radius: 6px; cursor: pointer; -webkit-app-region: no-drag;
|
||||
}
|
||||
.copyBtn:hover { background: rgba(125,211,252,0.2); }
|
||||
.msgCopy {
|
||||
position: absolute; top: 4px; right: 4px; width: 22px; height: 22px; border: none;
|
||||
background: rgba(255,255,255,0.06); color: #9ca3af; border-radius: 6px; cursor: pointer;
|
||||
font-size: 12px; opacity: 0; transition: opacity 0.12s; -webkit-app-region: no-drag;
|
||||
}
|
||||
.msg.assistant .bubble:hover .msgCopy { opacity: 1; }
|
||||
.msgCopy:hover { background: rgba(255,255,255,0.14); color: #e5e7eb; }
|
||||
|
||||
/* ---- Hologramm-Look (Holo-Lucy): Silhouetten-Glow + Scanlines + Cyan-Tint ---- */
|
||||
/* drop-shadow folgt der Alpha-Silhouette des transparenten WebGL-Canvas -> billiger „Bloom"-Halo */
|
||||
.avatarCanvas.holo canvas {
|
||||
filter: drop-shadow(0 0 5px rgba(90,222,255,0.85)) drop-shadow(0 0 15px rgba(56,190,255,0.5));
|
||||
}
|
||||
.holoTint {
|
||||
position: absolute; inset: 0; z-index: 2; pointer-events: none; mix-blend-mode: screen;
|
||||
background: radial-gradient(ellipse 60% 55% at 50% 42%, rgba(60,200,255,0.12), rgba(30,120,255,0.05) 62%, transparent 82%);
|
||||
}
|
||||
.holoScan {
|
||||
position: absolute; inset: 0; z-index: 3; pointer-events: none; mix-blend-mode: screen;
|
||||
background: repeating-linear-gradient(0deg, rgba(120,230,255,0.07) 0px, rgba(120,230,255,0.07) 1px, transparent 2px, transparent 4px);
|
||||
animation: holoFlicker 4s infinite;
|
||||
}
|
||||
@keyframes holoFlicker {
|
||||
0%, 100% { opacity: 0.5; } 48% { opacity: 0.46; } 50% { opacity: 0.72; } 52% { opacity: 0.45; }
|
||||
70% { opacity: 0.55; } 71% { opacity: 0.4; } 72% { opacity: 0.55; } 92% { opacity: 0.6; }
|
||||
}
|
||||
.holoToggle {
|
||||
position: absolute; top: 6px; right: 6px; z-index: 6; -webkit-app-region: no-drag;
|
||||
width: 26px; height: 24px; border-radius: 7px; cursor: pointer; font-size: 13px;
|
||||
border: 1px solid rgba(125,211,252,0.35); background: rgba(12,12,20,0.5); color: #7dd3fc;
|
||||
}
|
||||
.holoToggle:hover { background: rgba(125,211,252,0.2); }
|
||||
.holoToggle.on { background: rgba(125,211,252,0.28); color: #bae6fd; }
|
||||
@@ -1,18 +0,0 @@
|
||||
// Box-STT-Adapter — Spracherkennung läuft auf der Box (MC2 /api/voice/stt -> Parakeet,
|
||||
// whisper als Fallback). Der Dateiname muss zum Blob-Typ passen: der Sidecar wählt die
|
||||
// Dekodierung nach Suffix (Silero-VAD liefert WAV, Push-to-Talk weiterhin webm/opus).
|
||||
import { BOX_URL } from "../../config"
|
||||
import type { SttEngine } from "../types"
|
||||
|
||||
export class BoxSttEngine implements SttEngine {
|
||||
readonly id = "box-stt"
|
||||
constructor(private readonly baseUrl: string = BOX_URL) {}
|
||||
|
||||
async transcribe(audio: Blob): Promise<string> {
|
||||
const fd = new FormData()
|
||||
fd.append("audio", audio, audio.type.includes("wav") ? "rec.wav" : "rec.webm")
|
||||
const r = await fetch(`${this.baseUrl}/api/voice/stt`, { method: "POST", body: fd })
|
||||
if (!r.ok) throw new Error(`STT ${r.status}`)
|
||||
return (await r.json()).text?.trim() || ""
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
// Pocket-TTS-Adapter — kapselt den lokalen pocket_server (CPU, vom Main-Prozess gespawnt).
|
||||
// Endpunkte: POST /tts {text}->WAV, POST /tts/stream {text}->PCM16 (+ X-Sample-Rate), GET /health.
|
||||
import { TTS_URL } from "../../config"
|
||||
import type { TtsEngine, TtsStreamHandle } from "../types"
|
||||
|
||||
export class PocketTtsEngine implements TtsEngine {
|
||||
readonly id = "pocket"
|
||||
constructor(private readonly baseUrl: string = TTS_URL) {}
|
||||
|
||||
async synthesize(text: string): Promise<ArrayBuffer> {
|
||||
const r = await fetch(`${this.baseUrl}/tts`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }),
|
||||
})
|
||||
if (!r.ok) throw new Error(`TTS ${r.status}`)
|
||||
return r.arrayBuffer()
|
||||
}
|
||||
|
||||
async synthesizeStream(text: string): Promise<TtsStreamHandle> {
|
||||
const r = await fetch(`${this.baseUrl}/tts/stream`, {
|
||||
method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }),
|
||||
})
|
||||
if (!r.ok || !r.body) throw new Error(`TTS ${r.status}`)
|
||||
const sampleRate = Number(r.headers.get("X-Sample-Rate") || "24000")
|
||||
return { stream: r.body, sampleRate }
|
||||
}
|
||||
|
||||
// pocket_server serviert /health erst, wenn das Modell fertig geladen ist (FastAPI-lifespan).
|
||||
async waitReady(timeoutMs = 120_000): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const h = await (await fetch(`${this.baseUrl}/health`)).json()
|
||||
if (h.status === "ok") return true
|
||||
} catch { /* Dienst startet evtl. noch (Spawn + Modell-Load) */ }
|
||||
await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
// voice-core — öffentliche API + aktive Engine-Auswahl.
|
||||
// Aktuell fest: Pocket-TTS (lokal, Kyutai — macht auch Lucys Voice-Cloning) + Box-Whisper-STT.
|
||||
// TTS-Engines beschränkt auf pocket (live) + f5 (GPU-Option). ElevenLabs/Edge/Chatterbox/Piper
|
||||
// waren nur im alten Web-Pfad und sind hier bewusst NICHT dabei. Der Voice-Agent importiert NUR von hier.
|
||||
import { PocketTtsEngine } from "./engines/pocketTts"
|
||||
import { BoxSttEngine } from "./engines/boxStt"
|
||||
import type { SttEngine, TtsEngine } from "./types"
|
||||
|
||||
export const tts: TtsEngine = new PocketTtsEngine()
|
||||
export const stt: SttEngine = new BoxSttEngine()
|
||||
|
||||
export { SpeechScheduler } from "./speech"
|
||||
export type { AudioSink } from "./speech"
|
||||
export type { SttEngine, TtsEngine, TtsStreamHandle } from "./types"
|
||||
@@ -1,84 +0,0 @@
|
||||
// SpeechScheduler — Satz-Pipelining fürs Voll-Duplex-Gefühl (Etappe 2, Schritt 1).
|
||||
// Nimmt einzelne SÄTZE entgegen, während das Hirn noch streamt, und spricht sie lückenlos
|
||||
// nacheinander: erstes Audio nach dem ERSTEN Satz statt nach der ganzen Antwort.
|
||||
// Barge-in-fest über eine Generation (clear() verwirft laufende + wartende Sätze).
|
||||
import type { TtsEngine, TtsStreamHandle } from "./types"
|
||||
|
||||
// Perf-Diagnose: misst hörbare Stille + Synthese-Zeit je Satz. Abschalten via localStorage lucy_perf=0.
|
||||
const PERF = typeof localStorage !== "undefined" && localStorage.getItem("lucy_perf") !== "0"
|
||||
|
||||
/** Minimaler Audio-Ausgang (AudioQueue erfüllt diese Form) — hält voice-core von der UI entkoppelt. */
|
||||
export interface AudioSink {
|
||||
playPcmStream(stream: ReadableStream<Uint8Array>, sampleRate: number): Promise<void>
|
||||
}
|
||||
|
||||
export class SpeechScheduler {
|
||||
private pending: string[] = []
|
||||
private running = false
|
||||
private gen = 0
|
||||
private lastPlayEnd = 0 // Zeitstempel Ende des vorigen Satzes (für die „Stille davor"-Messung)
|
||||
/** true, sobald mindestens ein Satz seit dem letzten clear() gesprochen wurde (für Fehlererkennung). */
|
||||
playedAny = false
|
||||
/** Aktiv-Signal: true beim ersten Satz, false wenn alles gesprochen ist (treibt den „speaking"-Status). */
|
||||
onBusy?: (busy: boolean) => void
|
||||
|
||||
constructor(private readonly tts: TtsEngine, private readonly sink: AudioSink) {}
|
||||
|
||||
/** Einen fertigen Satz einreihen (leere werden ignoriert) und ggf. den Pump starten. */
|
||||
push(text: string): void {
|
||||
const t = text.trim()
|
||||
if (!t) return
|
||||
this.pending.push(t)
|
||||
if (!this.running) void this.pump()
|
||||
}
|
||||
|
||||
/** Barge-in / frischer Turn: laufende + wartende Sätze verwerfen. Der Audio-Sink wird separat
|
||||
* gestoppt (queue.clear()). Setzt die Fehler-/Erfolgs-Erkennung zurück. */
|
||||
clear(): void {
|
||||
this.gen++
|
||||
this.pending = []
|
||||
this.playedAny = false
|
||||
this.lastPlayEnd = 0
|
||||
}
|
||||
|
||||
/** Wartet, bis alle eingereihten Sätze gesprochen sind. */
|
||||
async idle(): Promise<void> {
|
||||
while (this.running || this.pending.length) await new Promise((r) => setTimeout(r, 40))
|
||||
}
|
||||
|
||||
private synth(text: string): Promise<TtsStreamHandle | null> {
|
||||
return this.tts.synthesizeStream(text).catch((e) => { console.error("TTS-Fehler:", e); return null })
|
||||
}
|
||||
|
||||
private async pump(): Promise<void> {
|
||||
if (this.running) return
|
||||
this.running = true
|
||||
this.onBusy?.(true)
|
||||
try {
|
||||
// SERIELL, KEIN Prefetch: der pocket_server ist Single-Instanz. synthesizeStream liefert den
|
||||
// Handle schon bei TTFB (Server generiert den Body noch) -> ein vorab angestoßenes Segment
|
||||
// würde den laufenden Lauf KONKURRIEREN lassen (gemessen bei Tool-Turns: gen 17s statt 5s,
|
||||
// TTFB-Spitze 14.7s). Gaplessness INNERHALB einer Antwort macht ohnehin der Server (Satz-Split
|
||||
// + _compress_gaps); hier werden nur die Segmente/Tool-Häppchen der Reihe nach gesprochen.
|
||||
let idx = 0
|
||||
while (this.pending.length) {
|
||||
const myGen = this.gen
|
||||
const tWait = performance.now()
|
||||
const handle = await this.synth(this.pending.shift()!)
|
||||
const synthWait = performance.now() - tWait
|
||||
if (myGen !== this.gen) { if (handle) handle.stream.cancel().catch(() => {}); continue } // ge-cleared -> Server sofort freigeben
|
||||
if (!handle) continue // Synthese fehlgeschlagen -> nächsten Satz versuchen
|
||||
const gap = this.lastPlayEnd ? tWait - this.lastPlayEnd : 0 // hörbare Stille seit dem letzten Satz
|
||||
const tPlay = performance.now()
|
||||
await this.sink.playPcmStream(handle.stream, handle.sampleRate)
|
||||
this.lastPlayEnd = performance.now()
|
||||
this.playedAny = true
|
||||
if (PERF) console.log(`[lucy-perf] TTS Satz #${idx}: Stille davor=${Math.round(gap)}ms · Synth-Warten=${Math.round(synthWait)}ms · Spieldauer=${Math.round(this.lastPlayEnd - tPlay)}ms`)
|
||||
idx++
|
||||
}
|
||||
} finally {
|
||||
this.running = false
|
||||
this.onBusy?.(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
// voice-core — Engine-Adapter-Interface (Etappe 0 des Greenfield-Umbaus).
|
||||
// Ziel: STT/TTS-Backends hinter EINEM Vertrag, damit pocket | f5 | piper | elevenlabs | edge
|
||||
// austauschbar sind (Config/Picker) und der Voice-Agent nicht mehr direkt an URLs klebt.
|
||||
|
||||
export interface SttEngine {
|
||||
/** Stabiler Bezeichner der Engine (z. B. "box-whisper"). */
|
||||
readonly id: string
|
||||
/** Audio (webm/opus/wav …) -> erkannter Text (leer, wenn nichts verstanden). */
|
||||
transcribe(audio: Blob): Promise<string>
|
||||
}
|
||||
|
||||
/** Ergebnis einer Streaming-Synthese: fortlaufender PCM16-mono-Stream + Samplerate. */
|
||||
export interface TtsStreamHandle {
|
||||
readonly stream: ReadableStream<Uint8Array>
|
||||
readonly sampleRate: number
|
||||
}
|
||||
|
||||
export interface TtsEngine {
|
||||
/** Stabiler Bezeichner der Engine (z. B. "pocket"). */
|
||||
readonly id: string
|
||||
/** Vollständige Synthese in einen dekodierbaren Audio-Buffer (z. B. WAV) — für Warm-up. */
|
||||
synthesize(text: string): Promise<ArrayBuffer>
|
||||
/** Streaming-PCM16-Synthese für niedrige Time-to-first-audio (die Live-Antwort). */
|
||||
synthesizeStream(text: string): Promise<TtsStreamHandle>
|
||||
/** Bereitschafts-/Warm-Gate: true, sobald die Engine antwortet (Modell evtl. lazy geladen). */
|
||||
waitReady(timeoutMs?: number): Promise<boolean>
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
@echo off
|
||||
title Lucy (Dev)
|
||||
cd /d "%~dp0"
|
||||
echo Starte Lucy ... (dieses Fenster offen lassen; schliessen = Lucy beenden)
|
||||
echo.
|
||||
call npm run dev
|
||||
echo.
|
||||
echo Lucy wurde beendet. Taste druecken zum Schliessen.
|
||||
pause >nul
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["node", "electron-vite/node"]
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
import re
|
||||
import site
|
||||
import time
|
||||
import jieba
|
||||
import torch
|
||||
import onnxruntime
|
||||
import soundfile as sf
|
||||
import numpy as np
|
||||
from pydub import AudioSegment
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
python_package_path = site.getsitepackages()[-1]
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-f5"
|
||||
vocab_path = BASE + r"\vocab_v1.txt"
|
||||
onnx_model_A = BASE + r"\onnx_f32\F5_Preprocess.onnx"
|
||||
onnx_model_B = BASE + r"\onnx_f32\F5_Transformer.onnx"
|
||||
onnx_model_C = BASE + r"\onnx_f32\F5_Decode.onnx"
|
||||
generated_audio = BASE + r"\bench_out.wav"
|
||||
test_in_english = True # eingebaute engl. Referenz -> kein Chinesisch/Pinyin; Tempo ist sprachunabhängig
|
||||
|
||||
if test_in_english:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_en.wav"
|
||||
ref_text = "Some call me nature, others call me mother nature."
|
||||
# realistischer Satz-Längen-Benchmark (wie eine echte Lucy-Antwort):
|
||||
gen_text = "Of course, Commander. I will restart the service, check the logs, and let you know once everything is running again."
|
||||
else:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_zh.wav" # The reference audio path.
|
||||
ref_text = "对，这就是我，万人敬仰的太乙真人。" # The ASR result of reference audio.
|
||||
gen_text = "对，这就是我，万人敬仰的大可奇奇。" # The target TTS.
|
||||
|
||||
|
||||
import os as _os
|
||||
ORT_Accelerate_Providers = [_os.environ.get("BENCH_PROVIDER", "DmlExecutionProvider")] # DML (9070 XT) oder CPUExecutionProvider
|
||||
# else keep empty.
|
||||
RANDOM_SEED = 9527 # Set seed to reproduce the generated audio
|
||||
NFE_STEP = int(_os.environ.get("BENCH_NFE", "32")) if (_os := __import__("os")) else 32 # via env testbar
|
||||
FUSE_NFE = 1 # Maintain the same values as the exported model.
|
||||
SPEED = 1.0 # Set for talking speed. Only works with dynamic_axes=True
|
||||
MAX_THREADS = 8 # Max CPU parallel threads.
|
||||
DEVICE_ID = 0 # The GPU id, default to 0.
|
||||
MODEL_SAMPLE_RATE = 24000 # Do not modify it.
|
||||
HOP_LENGTH = 256 # It affects the generated audio length and speech speed.
|
||||
|
||||
if "OpenVINOExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_type': 'CPU', # [CPU, NPU, GPU, GPU.0, GPU.1]]
|
||||
'precision': 'ACCURACY', # [FP32, FP16, ACCURACY]
|
||||
'num_of_threads': MAX_THREADS,
|
||||
'num_streams': 1,
|
||||
'enable_opencl_throttling': True,
|
||||
'enable_qdq_optimizer': False # Enable it carefully
|
||||
}
|
||||
]
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_id': DEVICE_ID,
|
||||
'gpu_mem_limit': 8 * 1024 * 1024 * 1024, # 8 GB
|
||||
'arena_extend_strategy': 'kNextPowerOfTwo',
|
||||
'cudnn_conv_algo_search': 'EXHAUSTIVE',
|
||||
'cudnn_conv_use_max_workspace': '1',
|
||||
'do_copy_in_default_stream': '1',
|
||||
'cudnn_conv1d_pad_to_nc1d': '1',
|
||||
'enable_cuda_graph': '0', # Set to '0' to avoid potential errors when enabled.
|
||||
'use_tf32': '0'
|
||||
}
|
||||
]
|
||||
else:
|
||||
# Please config by yourself for others providers.
|
||||
provider_options = None
|
||||
|
||||
|
||||
with open(vocab_path, "r", encoding="utf-8") as f:
|
||||
vocab_char_map = {}
|
||||
for i, char in enumerate(f):
|
||||
vocab_char_map[char[:-1]] = i
|
||||
vocab_size = len(vocab_char_map)
|
||||
|
||||
|
||||
# From the official code
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50) # CRITICAL
|
||||
jieba.initialize()
|
||||
|
||||
final_text_list = []
|
||||
custom_trans = str.maketrans(
|
||||
{";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"}
|
||||
) # add custom trans here, to address oov
|
||||
|
||||
def is_chinese(c):
|
||||
return (
|
||||
"\u3100" <= c <= "\u9fff" # common chinese characters
|
||||
)
|
||||
|
||||
for text in text_list:
|
||||
char_list = []
|
||||
text = text.translate(custom_trans)
|
||||
for seg in jieba.cut(text):
|
||||
seg_byte_len = len(bytes(seg, "UTF-8"))
|
||||
if seg_byte_len == len(seg): # if pure alphabets and symbols
|
||||
if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
|
||||
char_list.append(" ")
|
||||
char_list.extend(seg)
|
||||
elif polyphone and seg_byte_len == 3 * len(seg): # if pure east asian characters
|
||||
seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.append(seg_[i])
|
||||
else: # if mixed characters, alphabets and symbols
|
||||
for c in seg:
|
||||
if ord(c) < 256:
|
||||
char_list.extend(c)
|
||||
elif is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else:
|
||||
char_list.append(c)
|
||||
final_text_list.append(char_list)
|
||||
return final_text_list
|
||||
|
||||
|
||||
# From the official code
|
||||
def list_str_to_idx(
|
||||
text: list[str] | list[list[str]],
|
||||
vocab_char_map: dict[str, int], # {char: idx}
|
||||
padding_value=-1
|
||||
):
|
||||
get_idx = vocab_char_map.get
|
||||
list_idx_tensors = [torch.tensor([get_idx(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
text = torch.nn.utils.rnn.pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_to_int16(audio):
|
||||
max_val = np.max(np.abs(audio))
|
||||
scaling_factor = 32767.0 / max_val if max_val > 0 else 1.0
|
||||
return (audio * float(scaling_factor)).astype(np.int16)
|
||||
|
||||
|
||||
# ONNX Runtime settings
|
||||
onnxruntime.set_seed(RANDOM_SEED)
|
||||
session_opts = onnxruntime.SessionOptions()
|
||||
session_opts.log_severity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.log_verbosity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.inter_op_num_threads = MAX_THREADS # Run different nodes with num_threads. Set 0 for auto.
|
||||
session_opts.intra_op_num_threads = MAX_THREADS # Under the node, execute the operators with num_threads. Set 0 for auto.
|
||||
session_opts.enable_cpu_mem_arena = True # True for execute speed; False for less memory usage.
|
||||
session_opts.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
session_opts.add_session_config_entry("session.intra_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.inter_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.set_denormal_as_zero", "1")
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_A = onnxruntime.InferenceSession(onnx_model_A, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
model_type = ort_session_A._inputs_meta[0].type
|
||||
in_name_A = ort_session_A.get_inputs()
|
||||
out_name_A = ort_session_A.get_outputs()
|
||||
in_name_A0 = in_name_A[0].name
|
||||
in_name_A1 = in_name_A[1].name
|
||||
in_name_A2 = in_name_A[2].name
|
||||
out_name_A0 = out_name_A[0].name
|
||||
out_name_A1 = out_name_A[1].name
|
||||
out_name_A2 = out_name_A[2].name
|
||||
out_name_A3 = out_name_A[3].name
|
||||
out_name_A4 = out_name_A[4].name
|
||||
out_name_A5 = out_name_A[5].name
|
||||
out_name_A6 = out_name_A[6].name
|
||||
out_name_A7 = out_name_A[7].name
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
else:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=ORT_Accelerate_Providers, provider_options=provider_options)
|
||||
ORT_Accelerate_Providers = ort_session_B.get_providers()[0]
|
||||
# For Windows DirectML + Intel/AMD/Nvidia GPU,
|
||||
# pip install onnxruntime-directml --upgrade
|
||||
# ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=['DmlExecutionProvider'])
|
||||
print(f"\nUsable Providers: {ORT_Accelerate_Providers}")
|
||||
model_dtype = ort_session_B._inputs_meta[0].type
|
||||
in_name_B = ort_session_B.get_inputs()
|
||||
out_name_B = ort_session_B.get_outputs()
|
||||
in_name_B0 = in_name_B[0].name
|
||||
in_name_B1 = in_name_B[1].name
|
||||
in_name_B2 = in_name_B[2].name
|
||||
in_name_B3 = in_name_B[3].name
|
||||
in_name_B4 = in_name_B[4].name
|
||||
in_name_B5 = in_name_B[5].name
|
||||
in_name_B6 = in_name_B[6].name
|
||||
in_name_B7 = in_name_B[7].name
|
||||
out_name_B0 = out_name_B[0].name
|
||||
out_name_B1 = out_name_B[1].name
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_C = onnxruntime.InferenceSession(onnx_model_C, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
in_name_C = ort_session_C.get_inputs()
|
||||
out_name_C = ort_session_C.get_outputs()
|
||||
in_name_C0 = in_name_C[0].name
|
||||
in_name_C1 = in_name_C[1].name
|
||||
out_name_C0 = out_name_C[0].name
|
||||
|
||||
# Load the input audio
|
||||
print(f"\nReference Audio: {reference_audio}")
|
||||
audio = np.array(AudioSegment.from_file(reference_audio).set_channels(1).set_frame_rate(MODEL_SAMPLE_RATE).get_array_of_samples(), dtype=np.float32)
|
||||
audio = normalize_to_int16(audio)
|
||||
audio_len = len(audio)
|
||||
audio = audio.reshape(1, 1, -1)
|
||||
|
||||
zh_pause_punc = r"。，、；：？！"
|
||||
ref_text_len = len(ref_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, ref_text))
|
||||
gen_text_len = len(gen_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, gen_text))
|
||||
ref_audio_len = audio_len // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / SPEED)], dtype=np.int64)
|
||||
gen_text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(gen_text, vocab_char_map).numpy()
|
||||
time_step = np.array([0], dtype=np.int32)
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
device_type = 'cpu'
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers or "TensorrtExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'cuda'
|
||||
elif "DmlExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'dml'
|
||||
else:
|
||||
device_type = None
|
||||
|
||||
def run_pipeline():
|
||||
a_out = ort_session_A.run(
|
||||
[out_name_A0, out_name_A1, out_name_A2, out_name_A3, out_name_A4, out_name_A5, out_name_A6, out_name_A7],
|
||||
{in_name_A0: audio, in_name_A1: text_ids, in_name_A2: max_duration})
|
||||
noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ref_signal_len = a_out
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
if device_type:
|
||||
inputs = [onnxruntime.OrtValue.ortvalue_from_numpy(x, device_type, DEVICE_ID) for x in
|
||||
(noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ts)]
|
||||
outputs = [inputs[0], inputs[-1]]
|
||||
iob = ort_session_B.io_binding()
|
||||
for i in range(len(inputs)):
|
||||
iob.bind_ortvalue_input(name=in_name_B[i].name, ortvalue=inputs[i])
|
||||
for i in range(len(outputs)):
|
||||
iob.bind_ortvalue_output(name=out_name_B[i].name, ortvalue=outputs[i])
|
||||
for _ in range(0, NFE_STEP, FUSE_NFE):
|
||||
ort_session_B.run_with_iobinding(iob)
|
||||
noise = onnxruntime.OrtValue.numpy(iob.get_outputs()[0])
|
||||
else:
|
||||
for _ in range(0, NFE_STEP - 1, FUSE_NFE):
|
||||
noise, ts = ort_session_B.run(
|
||||
[out_name_B0, out_name_B1],
|
||||
{in_name_B0: noise, in_name_B1: rope_cos_q, in_name_B2: rope_sin_q, in_name_B3: rope_cos_k,
|
||||
in_name_B4: rope_sin_k, in_name_B5: cat_mel_text, in_name_B6: cat_mel_text_drop, in_name_B7: ts})
|
||||
return ort_session_C.run([out_name_C0], {in_name_C0: noise, in_name_C1: ref_signal_len})[0]
|
||||
|
||||
print(f"\nProvider={ORT_Accelerate_Providers} device_type={device_type} NFE={NFE_STEP}")
|
||||
print("Warmup (DML kompiliert beim 1. Lauf die Shader) ...")
|
||||
t0 = time.time(); _ = run_pipeline(); print(f" warmup gen = {time.time()-t0:.2f}s")
|
||||
best = 1e9
|
||||
for k in range(2):
|
||||
t0 = time.time(); gen = run_pipeline(); dt = time.time() - t0; best = min(best, dt)
|
||||
print(f" run {k+1}: gen = {dt:.2f}s")
|
||||
audio_s = gen.reshape(-1).shape[0] / MODEL_SAMPLE_RATE
|
||||
sf.write(generated_audio, gen.reshape(-1), MODEL_SAMPLE_RATE, format='WAVEX')
|
||||
rtf = best / max(audio_s, 0.01)
|
||||
print(f"\n=== ERGEBNIS NFE={NFE_STEP} === audio={audio_s:.2f}s gen(best)={best:.2f}s RTF={rtf:.2f} "
|
||||
f"({'REAL-TIME' if rtf < 1 else 'zu langsam'})")
|
||||
@@ -1,298 +0,0 @@
|
||||
import re
|
||||
import site
|
||||
import time
|
||||
import jieba
|
||||
import torch
|
||||
import onnxruntime
|
||||
import soundfile as sf
|
||||
import numpy as np
|
||||
from pydub import AudioSegment
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
python_package_path = site.getsitepackages()[-1]
|
||||
|
||||
vocab_path = "/home/DakeQQ/Downloads/F5TTS_v1_Base/vocab.txt" # The F5-TTS model vocab download path. URL: https://huggingface.co/SWivid/F5-TTS/tree/main/F5TTS_v1_Base
|
||||
onnx_model_A = "/home/DakeQQ/Downloads/F5_Optimized/F5_Preprocess.onnx" # The exported onnx model path.
|
||||
onnx_model_B = "/home/DakeQQ/Downloads/F5_Optimized/F5_Transformer.onnx" # The exported onnx model path.
|
||||
onnx_model_C = "/home/DakeQQ/Downloads/F5_Optimized/F5_Decode.onnx" # The exported onnx model path.
|
||||
generated_audio = "./generated_audio.wav"
|
||||
test_in_english = False
|
||||
|
||||
if test_in_english:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_en.wav"
|
||||
ref_text = "Some call me nature, others call me mother nature."
|
||||
gen_text = "Some call me Dake, others call me QQ."
|
||||
else:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_zh.wav" # The reference audio path.
|
||||
ref_text = "对，这就是我，万人敬仰的太乙真人。" # The ASR result of reference audio.
|
||||
gen_text = "对，这就是我，万人敬仰的大可奇奇。" # The target TTS.
|
||||
|
||||
|
||||
ORT_Accelerate_Providers = ['CPUExecutionProvider'] # If you have accelerate devices for : ['CUDAExecutionProvider', 'TensorrtExecutionProvider', 'CoreMLExecutionProvider', 'DmlExecutionProvider', 'OpenVINOExecutionProvider', 'ROCMExecutionProvider', 'MIGraphXExecutionProvider', 'AzureExecutionProvider']
|
||||
# else keep empty.
|
||||
RANDOM_SEED = 9527 # Set seed to reproduce the generated audio
|
||||
NFE_STEP = 32 # F5-TTS model setting, 0~31
|
||||
FUSE_NFE = 1 # Maintain the same values as the exported model.
|
||||
SPEED = 1.0 # Set for talking speed. Only works with dynamic_axes=True
|
||||
MAX_THREADS = 8 # Max CPU parallel threads.
|
||||
DEVICE_ID = 0 # The GPU id, default to 0.
|
||||
MODEL_SAMPLE_RATE = 24000 # Do not modify it.
|
||||
HOP_LENGTH = 256 # It affects the generated audio length and speech speed.
|
||||
|
||||
if "OpenVINOExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_type': 'CPU', # [CPU, NPU, GPU, GPU.0, GPU.1]]
|
||||
'precision': 'ACCURACY', # [FP32, FP16, ACCURACY]
|
||||
'num_of_threads': MAX_THREADS,
|
||||
'num_streams': 1,
|
||||
'enable_opencl_throttling': True,
|
||||
'enable_qdq_optimizer': False # Enable it carefully
|
||||
}
|
||||
]
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_id': DEVICE_ID,
|
||||
'gpu_mem_limit': 8 * 1024 * 1024 * 1024, # 8 GB
|
||||
'arena_extend_strategy': 'kNextPowerOfTwo',
|
||||
'cudnn_conv_algo_search': 'EXHAUSTIVE',
|
||||
'cudnn_conv_use_max_workspace': '1',
|
||||
'do_copy_in_default_stream': '1',
|
||||
'cudnn_conv1d_pad_to_nc1d': '1',
|
||||
'enable_cuda_graph': '0', # Set to '0' to avoid potential errors when enabled.
|
||||
'use_tf32': '0'
|
||||
}
|
||||
]
|
||||
else:
|
||||
# Please config by yourself for others providers.
|
||||
provider_options = None
|
||||
|
||||
|
||||
with open(vocab_path, "r", encoding="utf-8") as f:
|
||||
vocab_char_map = {}
|
||||
for i, char in enumerate(f):
|
||||
vocab_char_map[char[:-1]] = i
|
||||
vocab_size = len(vocab_char_map)
|
||||
|
||||
|
||||
# From the official code
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50) # CRITICAL
|
||||
jieba.initialize()
|
||||
|
||||
final_text_list = []
|
||||
custom_trans = str.maketrans(
|
||||
{";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"}
|
||||
) # add custom trans here, to address oov
|
||||
|
||||
def is_chinese(c):
|
||||
return (
|
||||
"\u3100" <= c <= "\u9fff" # common chinese characters
|
||||
)
|
||||
|
||||
for text in text_list:
|
||||
char_list = []
|
||||
text = text.translate(custom_trans)
|
||||
for seg in jieba.cut(text):
|
||||
seg_byte_len = len(bytes(seg, "UTF-8"))
|
||||
if seg_byte_len == len(seg): # if pure alphabets and symbols
|
||||
if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
|
||||
char_list.append(" ")
|
||||
char_list.extend(seg)
|
||||
elif polyphone and seg_byte_len == 3 * len(seg): # if pure east asian characters
|
||||
seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.append(seg_[i])
|
||||
else: # if mixed characters, alphabets and symbols
|
||||
for c in seg:
|
||||
if ord(c) < 256:
|
||||
char_list.extend(c)
|
||||
elif is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else:
|
||||
char_list.append(c)
|
||||
final_text_list.append(char_list)
|
||||
return final_text_list
|
||||
|
||||
|
||||
# From the official code
|
||||
def list_str_to_idx(
|
||||
text: list[str] | list[list[str]],
|
||||
vocab_char_map: dict[str, int], # {char: idx}
|
||||
padding_value=-1
|
||||
):
|
||||
get_idx = vocab_char_map.get
|
||||
list_idx_tensors = [torch.tensor([get_idx(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
text = torch.nn.utils.rnn.pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_to_int16(audio):
|
||||
max_val = np.max(np.abs(audio))
|
||||
scaling_factor = 32767.0 / max_val if max_val > 0 else 1.0
|
||||
return (audio * float(scaling_factor)).astype(np.int16)
|
||||
|
||||
|
||||
# ONNX Runtime settings
|
||||
onnxruntime.set_seed(RANDOM_SEED)
|
||||
session_opts = onnxruntime.SessionOptions()
|
||||
session_opts.log_severity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.log_verbosity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.inter_op_num_threads = MAX_THREADS # Run different nodes with num_threads. Set 0 for auto.
|
||||
session_opts.intra_op_num_threads = MAX_THREADS # Under the node, execute the operators with num_threads. Set 0 for auto.
|
||||
session_opts.enable_cpu_mem_arena = True # True for execute speed; False for less memory usage.
|
||||
session_opts.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
session_opts.add_session_config_entry("session.intra_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.inter_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.set_denormal_as_zero", "1")
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_A = onnxruntime.InferenceSession(onnx_model_A, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
model_type = ort_session_A._inputs_meta[0].type
|
||||
in_name_A = ort_session_A.get_inputs()
|
||||
out_name_A = ort_session_A.get_outputs()
|
||||
in_name_A0 = in_name_A[0].name
|
||||
in_name_A1 = in_name_A[1].name
|
||||
in_name_A2 = in_name_A[2].name
|
||||
out_name_A0 = out_name_A[0].name
|
||||
out_name_A1 = out_name_A[1].name
|
||||
out_name_A2 = out_name_A[2].name
|
||||
out_name_A3 = out_name_A[3].name
|
||||
out_name_A4 = out_name_A[4].name
|
||||
out_name_A5 = out_name_A[5].name
|
||||
out_name_A6 = out_name_A[6].name
|
||||
out_name_A7 = out_name_A[7].name
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
else:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=ORT_Accelerate_Providers, provider_options=provider_options)
|
||||
ORT_Accelerate_Providers = ort_session_B.get_providers()[0]
|
||||
# For Windows DirectML + Intel/AMD/Nvidia GPU,
|
||||
# pip install onnxruntime-directml --upgrade
|
||||
# ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=['DmlExecutionProvider'])
|
||||
print(f"\nUsable Providers: {ORT_Accelerate_Providers}")
|
||||
model_dtype = ort_session_B._inputs_meta[0].type
|
||||
in_name_B = ort_session_B.get_inputs()
|
||||
out_name_B = ort_session_B.get_outputs()
|
||||
in_name_B0 = in_name_B[0].name
|
||||
in_name_B1 = in_name_B[1].name
|
||||
in_name_B2 = in_name_B[2].name
|
||||
in_name_B3 = in_name_B[3].name
|
||||
in_name_B4 = in_name_B[4].name
|
||||
in_name_B5 = in_name_B[5].name
|
||||
in_name_B6 = in_name_B[6].name
|
||||
in_name_B7 = in_name_B[7].name
|
||||
out_name_B0 = out_name_B[0].name
|
||||
out_name_B1 = out_name_B[1].name
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_C = onnxruntime.InferenceSession(onnx_model_C, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
in_name_C = ort_session_C.get_inputs()
|
||||
out_name_C = ort_session_C.get_outputs()
|
||||
in_name_C0 = in_name_C[0].name
|
||||
in_name_C1 = in_name_C[1].name
|
||||
out_name_C0 = out_name_C[0].name
|
||||
|
||||
# Load the input audio
|
||||
print(f"\nReference Audio: {reference_audio}")
|
||||
audio = np.array(AudioSegment.from_file(reference_audio).set_channels(1).set_frame_rate(MODEL_SAMPLE_RATE).get_array_of_samples(), dtype=np.float32)
|
||||
audio = normalize_to_int16(audio)
|
||||
audio_len = len(audio)
|
||||
audio = audio.reshape(1, 1, -1)
|
||||
|
||||
zh_pause_punc = r"。，、；：？！"
|
||||
ref_text_len = len(ref_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, ref_text))
|
||||
gen_text_len = len(gen_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, gen_text))
|
||||
ref_audio_len = audio_len // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / SPEED)], dtype=np.int64)
|
||||
gen_text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(gen_text, vocab_char_map).numpy()
|
||||
time_step = np.array([0], dtype=np.int32)
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
device_type = 'cpu'
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers or "TensorrtExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'cuda'
|
||||
elif "DmlExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'dml'
|
||||
else:
|
||||
device_type = None
|
||||
|
||||
print("\n\nRun F5-TTS by ONNX Runtime.")
|
||||
start_count = time.time()
|
||||
noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ref_signal_len = ort_session_A.run(
|
||||
[out_name_A0, out_name_A1, out_name_A2, out_name_A3, out_name_A4, out_name_A5, out_name_A6, out_name_A7],
|
||||
{
|
||||
in_name_A0: audio,
|
||||
in_name_A1: text_ids,
|
||||
in_name_A2: max_duration
|
||||
})
|
||||
|
||||
if device_type:
|
||||
inputs = [
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(noise, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_cos_q, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_sin_q, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_cos_k, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_sin_k, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(cat_mel_text, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(cat_mel_text_drop, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(time_step, device_type, DEVICE_ID)
|
||||
]
|
||||
outputs = [
|
||||
inputs[0],
|
||||
inputs[-1]
|
||||
]
|
||||
|
||||
io_binding = ort_session_B.io_binding()
|
||||
for i in range(len(inputs)):
|
||||
io_binding.bind_ortvalue_input(
|
||||
name=in_name_B[i].name,
|
||||
ortvalue=inputs[i]
|
||||
)
|
||||
for i in range(len(outputs)):
|
||||
io_binding.bind_ortvalue_output(
|
||||
name=out_name_B[i].name,
|
||||
ortvalue=outputs[i]
|
||||
)
|
||||
|
||||
print("NFE_STEP: 0")
|
||||
for i in range(0, NFE_STEP, FUSE_NFE):
|
||||
ort_session_B.run_with_iobinding(io_binding)
|
||||
print(f"NFE_STEP: {i + FUSE_NFE}")
|
||||
noise = onnxruntime.OrtValue.numpy(io_binding.get_outputs()[0])
|
||||
else:
|
||||
print("NFE_STEP: 0")
|
||||
for i in range(0, NFE_STEP - 1, FUSE_NFE):
|
||||
noise, time_step = ort_session_B.run(
|
||||
[out_name_B0, out_name_B1],
|
||||
{
|
||||
in_name_B0: noise,
|
||||
in_name_B1: rope_cos_q,
|
||||
in_name_B2: rope_sin_q,
|
||||
in_name_B3: rope_cos_k,
|
||||
in_name_B4: rope_sin_k,
|
||||
in_name_B5: cat_mel_text,
|
||||
in_name_B6: cat_mel_text_drop,
|
||||
in_name_B7: time_step
|
||||
})
|
||||
print(f"NFE_STEP: {i + FUSE_NFE}")
|
||||
|
||||
generated_signal = ort_session_C.run(
|
||||
[out_name_C0],
|
||||
{
|
||||
in_name_C0: noise,
|
||||
in_name_C1: ref_signal_len
|
||||
})[0]
|
||||
end_count = time.time()
|
||||
|
||||
# Save to audio
|
||||
sf.write(generated_audio, generated_signal.reshape(-1), MODEL_SAMPLE_RATE, format='WAVEX')
|
||||
print(f"\nAudio generation is complete.\n\nONNXRuntime Time Cost in Seconds:\n{end_count - start_count:.3f}")
|
||||
@@ -1,196 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Lucy-Stimme v2: F5-TTS (deutsch) via ONNX Runtime + DirectML (9070 XT, nativ Windows, kein ROCm).
|
||||
Non-autoregressiv -> keine Kollaps-/Wiederhol-/Männerstimmen-Fehler wie pocket. Satzweises Streaming.
|
||||
Modelliert nach dem verifizierten DML-Benchmark (bench_dml.py) + Export_F5-Preprocessing + pocket_server-Struktur."""
|
||||
import os, io, re, time, threading, logging
|
||||
import numpy as np, soundfile as sf, librosa, jieba, torch
|
||||
import onnxruntime as ort
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import Response, JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
log = logging.getLogger("lucy-f5"); logging.basicConfig(level=logging.INFO)
|
||||
BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
ONNX_DIR = os.environ.get("LUCY_F5_ONNX", os.path.join(BASE, "onnx_de"))
|
||||
VOCAB = os.environ.get("LUCY_F5_VOCAB", os.path.join(BASE, "vocab.txt"))
|
||||
REF_WAV = os.environ.get("LUCY_F5_REF", os.path.join(BASE, "lucy_ref.wav"))
|
||||
REF_TXT = os.environ.get("LUCY_F5_REF_TXT", os.path.join(BASE, "lucy_ref.txt"))
|
||||
PROVIDER = os.environ.get("LUCY_F5_PROVIDER", "DmlExecutionProvider")
|
||||
NFE_STEP = int(os.environ.get("LUCY_F5_NFE", "32")) # MUSS zum Export passen (Zeitplan ist eingebacken)
|
||||
TARGET_RMS = float(os.environ.get("LUCY_TARGET_RMS", "0.09"))
|
||||
SR = 24000; HOP_LENGTH = 256
|
||||
STATE, LOCK = {}, threading.Lock()
|
||||
|
||||
# ---- Text-Preprocessing (aus Export_F5.py; für Deutsch laufen Nicht-CJK-Zeichen einfach durch) ----
|
||||
def _load_vocab(path):
|
||||
m = {}
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for i, ch in enumerate(f):
|
||||
m[ch[:-1]] = i
|
||||
return m
|
||||
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50); jieba.initialize()
|
||||
out, trans = [], str.maketrans({";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"})
|
||||
def is_zh(c): return "" <= c <= "鿿"
|
||||
for text in text_list:
|
||||
cl = []; text = text.translate(trans)
|
||||
for seg in jieba.cut(text):
|
||||
blen = len(bytes(seg, "UTF-8"))
|
||||
if blen == len(seg):
|
||||
if cl and blen > 1 and cl[-1] not in " :'\"": cl.append(" ")
|
||||
cl.extend(seg)
|
||||
elif polyphone and blen == 3 * len(seg):
|
||||
pin = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_zh(c): cl.append(" ")
|
||||
cl.append(pin[i])
|
||||
else:
|
||||
for c in seg:
|
||||
if ord(c) < 256: cl.extend(c)
|
||||
elif is_zh(c): cl.append(" "); cl.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else: cl.append(c)
|
||||
out.append(cl)
|
||||
return out
|
||||
|
||||
def list_str_to_idx(text, vocab_map, padding_value=-1):
|
||||
get = vocab_map.get
|
||||
tensors = [torch.tensor([get(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
return torch.nn.utils.rnn.pad_sequence(tensors, padding_value=padding_value, batch_first=True).numpy()
|
||||
|
||||
_ZH_PUNC = r"。，、；：？！"
|
||||
def _text_len(s): return len(s.encode("utf-8")) + 3 * len(re.findall(_ZH_PUNC, s))
|
||||
|
||||
# ---- Satz-Splitter (wie pocket) ----
|
||||
_SENT_RX = re.compile(r".+?(?:[.!?…]+(?:\s|$)|$)", re.S)
|
||||
def split_sentences(text, min_len=30):
|
||||
parts = [m.group(0).strip() for m in _SENT_RX.finditer(text.strip())]
|
||||
out = []
|
||||
for p in parts:
|
||||
if not p: continue
|
||||
if out and len(out[-1]) < min_len: out[-1] = f"{out[-1]} {p}"
|
||||
else: out.append(p)
|
||||
return out or [text.strip()]
|
||||
|
||||
def cleanup(a, sr):
|
||||
"""F5-Output putzen: Stille-Trim hinten, RMS-Norm auf TARGET_RMS, Peak-Clamp, 80ms-Pads."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if a.size == 0: return a
|
||||
rev, _ = librosa.effects.trim(a[::-1], top_db=40); a = rev[::-1] if rev.size else a
|
||||
yt, _ = librosa.effects.trim(a, top_db=40); a = yt if yt.size else a
|
||||
rms = float(np.sqrt(np.mean(a ** 2))) or 1e-9
|
||||
a = a * (TARGET_RMS / rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > 0.95: a = a * (0.95 / peak)
|
||||
fi = min(int(0.008 * sr), a.size // 2)
|
||||
if fi > 0:
|
||||
a[:fi] *= np.linspace(0., 1., fi, dtype=np.float32); a[-fi:] *= np.linspace(1., 0., fi, dtype=np.float32)
|
||||
pad = np.zeros(int(0.08 * sr), dtype=np.float32)
|
||||
return np.concatenate([pad, a, pad])
|
||||
|
||||
def _to_pcm16(a):
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
np.clip(a, -0.95, 0.95, out=a)
|
||||
return (a * 32767.0).astype("<i2").tobytes()
|
||||
|
||||
# ---- ONNX-Inferenz (A=Preprocess CPU, B=Transformer DML+io_binding, C=Decode CPU) ----
|
||||
def _infer(gen_text: str) -> np.ndarray:
|
||||
s = STATE
|
||||
ref_text = s["ref_text"]
|
||||
rt_len = _text_len(ref_text); gt_len = max(_text_len(gen_text), 1)
|
||||
ref_audio_len = s["ref_audio"].shape[-1] // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / rt_len * gt_len)], dtype=np.int64)
|
||||
text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(text, s["vocab"])
|
||||
A = s["A"].run(s["A_out"], {s["A_in"][0]: s["ref_audio"], s["A_in"][1]: text_ids, s["A_in"][2]: max_duration})
|
||||
noise, rcq, rsq, rck, rsk, cmt, cmtd, ref_signal_len = A
|
||||
dev = s["dev"]
|
||||
if dev: # DirectML/CUDA: io_binding, Tensoren GPU-resident über die NFE-Schleife
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
ins = [ort.OrtValue.ortvalue_from_numpy(x, dev, 0) for x in (noise, rcq, rsq, rck, rsk, cmt, cmtd, ts)]
|
||||
outs = [ins[0], ins[-1]]
|
||||
iob = s["B"].io_binding()
|
||||
for i in range(len(ins)): iob.bind_ortvalue_input(name=s["B_in"][i], ortvalue=ins[i])
|
||||
for i in range(len(outs)): iob.bind_ortvalue_output(name=s["B_out"][i], ortvalue=outs[i])
|
||||
for _ in range(0, NFE_STEP, 1): s["B"].run_with_iobinding(iob)
|
||||
noise = ort.OrtValue.numpy(iob.get_outputs()[0])
|
||||
else:
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
for _ in range(0, NFE_STEP - 1, 1):
|
||||
noise, ts = s["B"].run(s["B_out"], {s["B_in"][0]: noise, s["B_in"][1]: rcq, s["B_in"][2]: rsq,
|
||||
s["B_in"][3]: rck, s["B_in"][4]: rsk, s["B_in"][5]: cmt, s["B_in"][6]: cmtd, s["B_in"][7]: ts})
|
||||
out = s["C"].run([s["C_out"]], {s["C_in"][0]: noise, s["C_in"][1]: ref_signal_len})[0]
|
||||
a = np.asarray(out).reshape(-1).astype(np.float32)
|
||||
if a.dtype != np.float32 or np.max(np.abs(a)) > 1.5: # int16-Decoder -> auf float
|
||||
a = a / 32768.0
|
||||
return a
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
t0 = time.time(); log.info("Lade F5 ONNX (%s) ...", PROVIDER)
|
||||
so = ort.SessionOptions(); so.log_severity_level = 4
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
A = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Preprocess.onnx"), so, providers=["CPUExecutionProvider"])
|
||||
B = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Transformer.onnx"), so, providers=[PROVIDER])
|
||||
C = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Decode.onnx"), so, providers=["CPUExecutionProvider"])
|
||||
prov = B.get_providers()[0]
|
||||
dev = "dml" if "Dml" in prov else ("cuda" if "CUDA" in prov or "Tensorrt" in prov else None)
|
||||
# Lucy-Referenz als int16 laden (Decoder/Preprocess erwartet int16-Pfad)
|
||||
ref, _sr = sf.read(REF_WAV, dtype="float32", always_2d=False)
|
||||
ref = np.asarray(ref, dtype=np.float32).reshape(-1)
|
||||
if _sr != SR: ref = librosa.resample(ref, orig_sr=_sr, target_sr=SR)
|
||||
mx = np.max(np.abs(ref)) or 1.0
|
||||
ref_i16 = (ref * (32767.0 / mx)).astype(np.int16).reshape(1, 1, -1)
|
||||
STATE.update(
|
||||
A=A, B=B, C=C, dev=dev, prov=prov,
|
||||
A_in=[i.name for i in A.get_inputs()], A_out=[o.name for o in A.get_outputs()],
|
||||
B_in=[i.name for i in B.get_inputs()], B_out=[o.name for o in B.get_outputs()],
|
||||
C_in=[i.name for i in C.get_inputs()], C_out=C.get_outputs()[0].name,
|
||||
vocab=_load_vocab(VOCAB), ref_audio=ref_i16,
|
||||
ref_text=open(REF_TXT, encoding="utf-8").read().strip(),
|
||||
)
|
||||
log.info("Lucy-F5 bereit in %.1fs (Provider=%s, dev=%s, NFE=%d)", time.time() - t0, prov, dev, NFE_STEP)
|
||||
yield
|
||||
STATE.clear()
|
||||
|
||||
app = FastAPI(title="Lucy TTS (F5/DirectML)", lifespan=lifespan)
|
||||
|
||||
class Req(BaseModel):
|
||||
text: str
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok" if "A" in STATE else "loading", "engine": "f5-tts",
|
||||
"provider": STATE.get("prov"), "nfe": NFE_STEP, "sr": SR}
|
||||
|
||||
@app.post("/tts")
|
||||
def tts(req: Req):
|
||||
if "A" not in STATE: return JSONResponse({"error": "loading"}, status_code=503)
|
||||
t0 = time.time()
|
||||
parts = []
|
||||
with LOCK:
|
||||
for sent in split_sentences(req.text):
|
||||
parts.append(cleanup(_infer(sent), SR))
|
||||
a = np.concatenate(parts) if parts else np.zeros(0, np.float32)
|
||||
buf = io.BytesIO(); sf.write(buf, a, SR, format="WAV", subtype="PCM_16"); buf.seek(0)
|
||||
dur = a.size / SR; gen = time.time() - t0
|
||||
log.info("/tts %dZ audio=%.1fs gen=%.1fs rtf=%.2f", len(req.text), dur, gen, gen / max(dur, 0.01))
|
||||
return Response(buf.read(), media_type="audio/wav",
|
||||
headers={"X-Audio-Seconds": f"{dur:.2f}", "X-Gen-Seconds": f"{gen:.2f}"})
|
||||
|
||||
@app.post("/tts/stream")
|
||||
def tts_stream(req: Req):
|
||||
if "A" not in STATE: return JSONResponse({"error": "loading"}, status_code=503)
|
||||
sentences = split_sentences(req.text)
|
||||
def pcm():
|
||||
t0 = time.time(); total = 0; first = True
|
||||
with LOCK:
|
||||
for sent in sentences:
|
||||
a = cleanup(_infer(sent), SR); total += a.size
|
||||
if first: log.info("/tts/stream TTFB=%.2fs (%d Sätze)", time.time() - t0, len(sentences)); first = False
|
||||
yield _to_pcm16(a)
|
||||
log.info("/tts/stream %dZ audio=%.1fs gen=%.1fs", len(req.text), total / SR, time.time() - t0)
|
||||
return StreamingResponse(pcm(), media_type="application/octet-stream", headers={"X-Sample-Rate": str(SR)})
|
||||
@@ -1,44 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""F5-TTS Qualitäts-/Tempo-Test (CPU): dt. Finetune + Lucy-Klon. Gleiche Sätze wie pocket -> A/B."""
|
||||
import os, time, soundfile as sf, numpy as np, torch
|
||||
# torchcodec/ffmpeg fehlt -> torchaudio.load/save auf soundfile umbiegen (wie bei OuteTTS-Patch)
|
||||
import torchaudio
|
||||
def _ta_load(path, *a, **k):
|
||||
data, sr = sf.read(str(path), dtype="float32", always_2d=True)
|
||||
return torch.from_numpy(data.T.copy()), sr
|
||||
def _ta_save(path, tensor, sr, *a, **k):
|
||||
arr = np.asarray(tensor.detach().cpu().numpy())
|
||||
sf.write(str(path), arr.T if arr.ndim == 2 else arr, sr)
|
||||
torchaudio.load = _ta_load
|
||||
torchaudio.save = _ta_save
|
||||
from f5_tts.api import F5TTS
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-f5"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_f5_test"; os.makedirs(OUT, exist_ok=True)
|
||||
ref = os.path.join(BASE, "lucy_ref.wav")
|
||||
ref_text = open(os.path.join(BASE, "lucy_ref.txt"), encoding="utf-8").read().strip()
|
||||
print("REF_TEXT:", ref_text[:80], flush=True)
|
||||
|
||||
t0 = time.time()
|
||||
f5 = F5TTS(model="F5TTS_Base",
|
||||
ckpt_file=os.path.join(BASE, "model_f5tts_german.safetensors"),
|
||||
vocab_file=os.path.join(BASE, "vocab.txt"), device="cpu")
|
||||
print(f"Modell geladen in {time.time()-t0:.1f}s (Vocoder evtl. erst geladen)", flush=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
for name, text in SENT.items():
|
||||
t0 = time.time()
|
||||
wav, sr, _ = f5.infer(ref_file=ref, ref_text=ref_text, gen_text=text,
|
||||
nfe_step=32, target_rms=0.1, remove_silence=True)
|
||||
dt = time.time() - t0
|
||||
wav = np.asarray(wav, dtype=np.float32).reshape(-1)
|
||||
peak = float(np.max(np.abs(wav))) # Peak-Limiter gegen Clipping (F5 traf 1.0)
|
||||
if peak > 0.95: wav = wav * (0.95 / peak)
|
||||
sf.write(os.path.join(OUT, f"{name}.wav"), wav, sr)
|
||||
secs = len(wav) / sr
|
||||
print(f"[{name:6}] gen={dt:6.1f}s audio={secs:5.1f}s RTF={dt/max(secs,0.01):5.2f}", flush=True)
|
||||
print("F5_TEST_DONE", flush=True)
|
||||
@@ -1 +0,0 @@
|
||||
Hallo, schön, dass du da bist. Ich bin deine persönliche Assistentin und begleite dich durch deinen Tag.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,12 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo B2-Test: seriell vs. 3 Worker (laedt mehrere Modelle - kann ein paar Minuten dauern)...
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" bench_b2.py > bench_b2.log 2>&1
|
||||
type bench_b2.log
|
||||
echo.
|
||||
echo ============================================================
|
||||
echo Fertig. WAVs in: %USERPROFILE%\Desktop\lucy_samples (b2_serial / b2_parallel_3w)
|
||||
echo ============================================================
|
||||
pause
|
||||
@@ -1,13 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo Erzeuge Lucy-Hoerproben (A1 safetensors-Cache + A2 Referenz-Cleaning)...
|
||||
echo Modell laedt offline aus dem lokalen Cache - das kann ein paar Minuten dauern.
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" make_samples.py > make_samples.log 2>&1
|
||||
type make_samples.log
|
||||
echo.
|
||||
echo ============================================================
|
||||
echo Fertig. Samples liegen in: %USERPROFILE%\Desktop\lucy_samples
|
||||
echo ============================================================
|
||||
pause
|
||||
@@ -1,13 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo Lucy-Startklar-Check: bootet die Stimme + rendert finale Samples auf den Desktop...
|
||||
echo (Modell laedt offline aus dem Cache - ein paar Sekunden)
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" lucy_ready.py > lucy_ready.log 2>&1
|
||||
type lucy_ready.log
|
||||
echo.
|
||||
echo ============================================================
|
||||
echo Samples: %USERPROFILE%\Desktop\lucy_samples (lucy_final_*)
|
||||
echo ============================================================
|
||||
pause
|
||||
@@ -1,12 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo B2-Sweep: WORKERS x THREADS finden (laedt viele Modelle - dauert ein paar Minuten)...
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" sweep_b2.py > sweep_b2.log 2>&1
|
||||
echo ============================================================
|
||||
type sweep_b2_result.json 2>nul
|
||||
echo.
|
||||
echo (Volllog: sweep_b2.log)
|
||||
echo ============================================================
|
||||
pause
|
||||
@@ -1,9 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo TTFB-Test (seriell): kurzer-erster-Chunk an/aus...
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" ttfb_test.py > ttfb_test.log 2>&1
|
||||
type ttfb_test.log
|
||||
echo.
|
||||
pause
|
||||
@@ -1,9 +0,0 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
cd /d "F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
echo Umlaut-Test: Logik-Check + Vorher/Nachher-Audio...
|
||||
echo.
|
||||
"ptts-venv\Scripts\python.exe" umlaut_test.py > umlaut_test.log 2>&1
|
||||
type umlaut_test.log
|
||||
echo.
|
||||
pause
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Stimm-RÖNTGEN: liest Wörter + Wort-genaue Zeiten + PAUSEN + Prosodie (F0/Tempo) aus einer WAV.
|
||||
So sieht man beim Tunen objektiv, wo Pocket zu lange Pausen macht (Komma!), nuschelt, oder monoton wird.
|
||||
|
||||
Nutzung: python analyze.py <datei.wav>
|
||||
"""
|
||||
import sys, numpy as np, soundfile as sf, librosa
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
p = sys.argv[1]
|
||||
a, sr = sf.read(p)
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
dur = len(a) / sr
|
||||
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
segs, _ = w.transcribe(p, language="de", beam_size=5, word_timestamps=True)
|
||||
words = [x for s in segs for x in (s.words or [])]
|
||||
txt = " ".join(x.word.strip() for x in words)
|
||||
|
||||
print(f"=== {p.split(chr(92))[-1]} ===")
|
||||
print(f"Dauer {dur:.2f}s | {len(words)} Wörter | Tempo {len(words)/max(dur,0.01):.1f} Wörter/s")
|
||||
print(f"TEXT: {txt}\n")
|
||||
|
||||
print("WORT-TIMING & PAUSE danach:")
|
||||
pauses = []
|
||||
for i, x in enumerate(words):
|
||||
gap = (words[i + 1].start - x.end) if i + 1 < len(words) else 0.0
|
||||
flag = " <=== LANG" if gap >= 0.30 else (" <- Pause" if gap >= 0.15 else "")
|
||||
if gap >= 0.15:
|
||||
pauses.append((x.word.strip(), gap))
|
||||
print(f" {x.start:5.2f}-{x.end:5.2f} {x.word.strip():16} Pause: {gap*1000:4.0f}ms{flag}")
|
||||
|
||||
# Rand-Stille
|
||||
env = np.abs(a) > 0.015
|
||||
lead = (np.argmax(env) / sr * 1000) if env.any() else 0
|
||||
tail = ((len(a) - 1 - np.argmax(env[::-1])) if env.any() else len(a))
|
||||
tail_ms = (len(a) - tail) / sr * 1000
|
||||
|
||||
# Prosodie: F0 (Tonhöhe) über stimmhafte Frames
|
||||
f0 = librosa.yin(a, fmin=80, fmax=400, sr=sr, frame_length=1024)
|
||||
f0v = f0[(f0 > 90) & (f0 < 380)]
|
||||
if f0v.size:
|
||||
med = float(np.median(f0v)); rng = float(np.percentile(f0v, 90) - np.percentile(f0v, 10))
|
||||
else:
|
||||
med = rng = 0.0
|
||||
|
||||
print(f"\nPAUSEN gesamt: {len(pauses)} (>=150ms) | längste: " +
|
||||
(", ".join(f'nach „{w_}\": {g*1000:.0f}ms' for w_, g in sorted(pauses, key=lambda t: -t[1])[:4]) or "keine"))
|
||||
print(f"RAND-STILLE: vorne {lead:.0f}ms, hinten {tail_ms:.0f}ms")
|
||||
print(f"TONHÖHE: median {med:.0f}Hz, Spanne {rng:.0f}Hz ({'monoton' if rng < 40 else 'lebendig' if rng > 90 else 'ok'})")
|
||||
@@ -1,67 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""bench_b2.py — verifiziert B2 (Worker-Pool) end-to-end mit den ECHTEN pocket_server-Funktionen.
|
||||
Misst die Wall-Clock einer langen, mehrsätzigen Antwort SERIELL (1 Modell) vs. 3 WORKER parallel,
|
||||
prüft die Reihenfolge (geordnete Ausgabe) und legt beide Ergebnis-WAVs auf den Desktop.
|
||||
"""
|
||||
import os, time
|
||||
import numpy as np, soundfile as sf, librosa
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pocket_tts import TTSModel
|
||||
import pocket_server as ps
|
||||
|
||||
DESK = os.path.join(os.path.expanduser("~"), "Desktop", "lucy_samples")
|
||||
os.makedirs(DESK, exist_ok=True)
|
||||
|
||||
LONG = ("Guten Morgen, Commander. Das nächtliche Backup ist sauber durchgelaufen. "
|
||||
"Es gab keine Fehler in den Protokollen. Der Dienst läuft stabil weiter. "
|
||||
"Ich habe die Modelle vorgewärmt und die Engine antwortet zügig. "
|
||||
"Wenn du möchtest, starte ich jetzt den Tagesbericht und fasse die offenen Punkte zusammen.")
|
||||
|
||||
|
||||
def _serial():
|
||||
m = TTSModel.load_model(language=ps.LANG, lsd_decode_steps=ps.LSD, temp=ps.TEMP,
|
||||
noise_clamp=ps.NOISE_CLAMP, quantize=ps.QUANTIZE)
|
||||
ref = ps._prep_ref()
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(ps.VOICE_ST)
|
||||
except Exception:
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
ref_audio, _ = librosa.load(ref, sr=m.sample_rate, mono=True)
|
||||
ps.STATE.clear()
|
||||
ps.STATE.update(m=m, vs=vs, sr=m.sample_rate, ref_fp=ps._fingerprint(ref_audio, m.sample_rate))
|
||||
sents = ps._split_sentences(LONG)
|
||||
t0 = time.time()
|
||||
parts = list(ps._gen_sentences_ordered(sents)) # pool=None -> serieller Zweig (LOCK)
|
||||
dt = time.time() - t0
|
||||
a = np.concatenate(parts)
|
||||
sf.write(os.path.join(DESK, "b2_serial.wav"), a, m.sample_rate)
|
||||
ps.STATE.clear(); del m
|
||||
return dt, a.size / 24000, len(sents)
|
||||
|
||||
|
||||
def _parallel(nworkers):
|
||||
ps.STATE.clear()
|
||||
pool = ProcessPoolExecutor(max_workers=nworkers, initializer=ps._worker_init)
|
||||
list(pool.map(ps._warmup, range(nworkers))) # alle Worker vorab hochfahren
|
||||
ref = ps._prep_ref()
|
||||
ref_audio, _ = librosa.load(ref, sr=24000, mono=True)
|
||||
ps.STATE.update(sr=24000, ref_fp=ps._fingerprint(ref_audio, 24000), pool=pool)
|
||||
sents = ps._split_sentences(LONG)
|
||||
t0 = time.time()
|
||||
parts = list(ps._gen_sentences_ordered(sents)) # pool -> parallel, geordnet
|
||||
dt = time.time() - t0
|
||||
a = np.concatenate(parts)
|
||||
sf.write(os.path.join(DESK, f"b2_parallel_{nworkers}w.wav"), a, 24000)
|
||||
pool.shutdown(wait=True); ps.STATE.clear()
|
||||
return dt, a.size / 24000, len(sents)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"Text: {len(LONG)} Zeichen")
|
||||
ds, audio_s, n = _serial()
|
||||
print(f"SERIELL : {ds:5.2f}s wall ({n} Saetze, {audio_s:4.1f}s Audio) -> b2_serial.wav")
|
||||
dp, _, _ = _parallel(3)
|
||||
print(f"3 WORKER: {dp:5.2f}s wall -> b2_parallel_3w.wav")
|
||||
if dp > 0:
|
||||
print(f"Speedup : {ds/dp:.2f}x (Audio gleich lang -> nur Generierzeit zaehlt)")
|
||||
print("B2_DONE")
|
||||
@@ -1,109 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""bench_lucy.py — A3/B1: misst Lucys Stimme (pocket-tts) über temp/lsd.
|
||||
|
||||
Metriken je Konfig: RTF, gen-Zeit und ROHE Kollaps-Rate (EINE Generierung, OHNE best-of-N-Gate)
|
||||
— also genau das, was der Stimm-Wächter aktuell wegbügeln muss (= Latenz-Treiber). Niedrigere
|
||||
temp sollte die rohe Kollaps-Rate senken -> weniger Regenerationen -> niedrigere effektive Latenz.
|
||||
|
||||
Nutzung (im ptts-venv der lucy-tts-Maschine):
|
||||
python bench_lucy.py # Default-Sweep
|
||||
python bench_lucy.py --reps 5
|
||||
python bench_lucy.py --temps 0.7,0.8,0.9 --lsds 8,10,12
|
||||
python bench_lucy.py --whisper # + Verständlichkeits-Check (faster-whisper)
|
||||
|
||||
Reuse: Kollaps-Logik (F0 + MFCC-Fingerabdruck) und cleanup() kommen 1:1 aus pocket_server.py,
|
||||
damit der Benchmark exakt die Produktions-Kriterien misst.
|
||||
"""
|
||||
import os, time, argparse, statistics as st
|
||||
import numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
import pocket_server as ps
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel": "Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die "
|
||||
"Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
|
||||
|
||||
def _voice_state(m):
|
||||
"""Voice-State laden: bevorzugt das gecachte safetensors (A1), sonst aus der Referenz klonen."""
|
||||
if os.path.exists(ps.VOICE_ST):
|
||||
try:
|
||||
return m.get_state_for_audio_prompt(ps.VOICE_ST)
|
||||
except Exception as e:
|
||||
print(f" (safetensors-Load fehlgeschlagen, klone aus Referenz: {e})")
|
||||
return m.get_state_for_audio_prompt(ps._prep_ref())
|
||||
|
||||
|
||||
def _raw_gen(m, vs, ref_fp, text, sr):
|
||||
"""EINE rohe Generierung (kein Gate). -> (gen_s, audio_s, male, sim, audio)."""
|
||||
t0 = time.time()
|
||||
audio = m.generate_audio(vs, ps.LEAD + text, frames_after_eos=ps.FEOS)
|
||||
gen = time.time() - t0
|
||||
a = audio.numpy() if hasattr(audio, "numpy") else np.asarray(audio)
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
f0 = ps._voiced_f0(a, sr)
|
||||
male = (f0 == f0) and f0 < ps.F0_FLOOR # NaN-sicher
|
||||
fp = ps._fingerprint(ps._crop_lead(a, sr), sr)
|
||||
sim = float(np.dot(ref_fp, fp)) if (ref_fp is not None and fp is not None) else 1.0
|
||||
return gen, a.size / sr, bool(male), sim, a
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--temps", default="0.7,0.8,0.9")
|
||||
ap.add_argument("--lsds", default="8,10")
|
||||
ap.add_argument("--reps", type=int, default=4)
|
||||
ap.add_argument("--whisper", action="store_true")
|
||||
args = ap.parse_args()
|
||||
temps = [float(x) for x in args.temps.split(",")]
|
||||
lsds = [int(x) for x in args.lsds.split(",")]
|
||||
|
||||
OUT = os.path.join(ps.BASE, "out_bench"); os.makedirs(OUT, exist_ok=True)
|
||||
ref = ps._prep_ref()
|
||||
print(f"LANG={ps.LANG} quantize={ps.QUANTIZE} noise_clamp={ps.NOISE_CLAMP} "
|
||||
f"F0_FLOOR={ps.F0_FLOOR} FP_FLOOR={ps.FP_FLOOR} reps={args.reps}")
|
||||
print(f"{'temp':>5} {'lsd':>4} | {'RTF':>5} {'gen/s':>6} | "
|
||||
f"{'collapse%':>9} {'sim_min':>7} {'sim_avg':>7}")
|
||||
rows = []
|
||||
for lsd in lsds:
|
||||
for temp in temps:
|
||||
m = TTSModel.load_model(language=ps.LANG, lsd_decode_steps=lsd, temp=temp,
|
||||
noise_clamp=ps.NOISE_CLAMP, quantize=ps.QUANTIZE)
|
||||
sr = m.sample_rate
|
||||
ref_audio, _ = librosa.load(ref, sr=sr, mono=True)
|
||||
ref_fp = ps._fingerprint(ref_audio, sr)
|
||||
vs = _voice_state(m)
|
||||
rtfs, gens, sims, collapses, n = [], [], [], 0, 0
|
||||
for name, text in SENT.items():
|
||||
for r in range(args.reps):
|
||||
gen, asec, male, sim, a = _raw_gen(m, vs, ref_fp, text, sr)
|
||||
rtfs.append(gen / max(asec, 0.01)); gens.append(gen); sims.append(sim)
|
||||
collapses += int(male or sim < ps.FP_FLOOR); n += 1
|
||||
if r == 0: # ein Sample je Satz zum Reinhören
|
||||
sf.write(os.path.join(OUT, f"t{temp}_l{lsd}_{name}.wav"), ps.cleanup(a, sr), sr)
|
||||
cr = 100.0 * collapses / max(n, 1)
|
||||
print(f"{temp:>5} {lsd:>4} | {st.mean(rtfs):>5.2f} {st.mean(gens):>6.2f} | "
|
||||
f"{cr:>8.1f}% {min(sims):>7.3f} {st.mean(sims):>7.3f}")
|
||||
rows.append((temp, lsd, st.mean(rtfs), cr, min(sims)))
|
||||
del m
|
||||
|
||||
best = sorted(rows, key=lambda x: (x[3], x[2]))[0] # min Kollaps, dann beste RTF
|
||||
print(f"\n>> Vorschlag: temp={best[0]} lsd={best[1]} "
|
||||
f"(collapse={best[3]:.1f}%, RTF={best[2]:.2f}) -> in pocket_server via "
|
||||
f"LUCY_TEMP / LUCY_LSD setzen.")
|
||||
|
||||
if args.whisper:
|
||||
print("\n=== Whisper-Verständlichkeit (faster-whisper small/int8) ===")
|
||||
from faster_whisper import WhisperModel
|
||||
import glob
|
||||
wm = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
for p in sorted(glob.glob(os.path.join(OUT, "*.wav"))):
|
||||
segs, _ = wm.transcribe(p, language="de", beam_size=5)
|
||||
print(os.path.basename(p), "::", " ".join(s.text.strip() for s in segs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,32 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Debug: warum greift der Onset-Crop auf dem 1s-Kopf des Streams nicht? + Roh-RMS für Gain-Kalibrierung."""
|
||||
import os, numpy as np, librosa, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
src,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
srt,_ = librosa.effects.trim(src, top_db=30); ref=os.path.join(BASE,"ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=0.9)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr = m.sample_rate
|
||||
LEAD = "Tja. "
|
||||
text = LEAD + "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu."
|
||||
|
||||
chunks = []
|
||||
for c in m.generate_audio_stream(vs, text, frames_after_eos=4):
|
||||
c = c.numpy() if hasattr(c,"numpy") else np.asarray(c)
|
||||
chunks.append(np.asarray(c,dtype=np.float32).reshape(-1))
|
||||
print(f"#chunks={len(chunks)} chunk_sizes_ms={[round(x.size/sr*1000) for x in chunks[:8]]}")
|
||||
full = np.concatenate(chunks)
|
||||
print(f"full dur={full.size/sr:.2f}s peak={np.abs(full).max():.3f} rms={np.sqrt(np.mean(full**2)):.3f}")
|
||||
|
||||
head = full[:int(1.0*sr)]
|
||||
for td in (45, 35, 25, 20):
|
||||
iv = librosa.effects.split(head, top_db=td)
|
||||
segs = [(round(s/sr,2), round(e/sr,2)) for s,e in iv]
|
||||
print(f"HEAD top_db={td}: {len(iv)} segs {segs}")
|
||||
# voller Onset-Crop (wie nicht-stream) zum Vergleich
|
||||
ivf = librosa.effects.split(full, top_db=35)
|
||||
print(f"FULL top_db=35: {len(ivf)} segs erste3={[(round(s/sr,2),round(e/sr,2)) for s,e in ivf[:3]]}")
|
||||
print("DBG_DONE")
|
||||
@@ -1,41 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Diagnose: schneidet das Modell selbst vorne/hinten ab, oder mein Trim?
|
||||
Misst Stille-Anteil am Anfang/Ende des ROHEN Outputs (vor jedem Trim)."""
|
||||
import os, time, numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = os.path.join(BASE, "out_diag"); os.makedirs(OUT, exist_ok=True)
|
||||
y, _ = librosa.load(os.path.join(BASE, "ref.mp3"), sr=24000, mono=True)
|
||||
yt, _ = librosa.effects.trim(y, top_db=30); ref = os.path.join(BASE, "ref.wav")
|
||||
sf.write(ref, yt[:int(18*24000)], 24000)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
|
||||
def lead_tail_silence(a, sr):
|
||||
"""Wie viel ms am Anfang/Ende liegen unter -40dB vom Peak (= 'Stille')?"""
|
||||
a = np.abs(np.asarray(a, dtype=np.float32).reshape(-1))
|
||||
if a.size == 0: return 0.0, 0.0, 0.0
|
||||
peak = a.max() + 1e-9
|
||||
thr = peak * (10 ** (-40/20)) # -40 dB
|
||||
above = np.where(a > thr)[0]
|
||||
if above.size == 0: return a.size/sr*1000, a.size/sr*1000, 0.0
|
||||
lead = above[0] / sr * 1000
|
||||
tail = (a.size - 1 - above[-1]) / sr * 1000
|
||||
return lead, tail, a.size/sr*1000
|
||||
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=4)
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
sr = m.sample_rate
|
||||
for name, text in SENT.items():
|
||||
for feos in [None, 4, 8]:
|
||||
a = m.generate_audio(vs, text, frames_after_eos=feos)
|
||||
a = a.numpy() if hasattr(a, "numpy") else np.asarray(a)
|
||||
a = a.reshape(-1)
|
||||
lead, tail, total = lead_tail_silence(a, sr)
|
||||
sf.write(os.path.join(OUT, f"raw_{name}_feos{feos}.wav"), a, sr)
|
||||
print(f"[{name:5} feos={str(feos):4}] total={total:7.1f}ms lead_sil={lead:6.1f}ms tail_sil={tail:6.1f}ms peak={np.abs(a).max():.3f}", flush=True)
|
||||
print("DIAG_DONE", flush=True)
|
||||
@@ -1,40 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Stabilität gegen Stimm-Kollaps: temp 0.9 mit/ohne noise_clamp. F0-Verteilung + Whisper + RTF.
|
||||
Mehr Samples, um einen Kollaps (tiefe F0) zu provozieren und zu sehen, ob noise_clamp ihn dämpft."""
|
||||
import os, time, numpy as np, librosa, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
src,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
srt,_ = librosa.effects.trim(src, top_db=30); ref=os.path.join(BASE,"ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
LEAD="Tja. "
|
||||
TEXTS = ["Hallo Commander, ich höre dich.","Natürlich, das mache ich sofort für dich.",
|
||||
"Die Hitze ist heute wirklich heftig.","Klar, ich kümmere mich gleich darum.",
|
||||
"Guten Morgen, Commander, alles sauber.","Kein Problem, ich erledige das jetzt.",
|
||||
"Das Backup lief ohne Fehler durch.","Verstanden, Commander, ich bleibe dran."]
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
|
||||
def f0_of(a, sr):
|
||||
f0,_,_ = librosa.pyin(a, fmin=80, fmax=400, sr=sr, frame_length=1024)
|
||||
v=f0[~np.isnan(f0)]; return float(np.median(v)) if v.size else float("nan")
|
||||
|
||||
def run(temp, nc):
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=temp, noise_clamp=nc)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr=m.sample_rate
|
||||
print(f"=== temp={temp} noise_clamp={nc} ===", flush=True)
|
||||
f0s=[]; rtfs=[]
|
||||
for i in range(12):
|
||||
t=TEXTS[i%len(TEXTS)]
|
||||
t0=time.time(); a=m.generate_audio(vs, LEAD+t, frames_after_eos=4); dt=time.time()-t0
|
||||
a=a.numpy() if hasattr(a,"numpy") else np.asarray(a); a=np.asarray(a,dtype=np.float32).reshape(-1)
|
||||
f0=f0_of(a,sr); f0s.append(f0); rtfs.append(dt/max(a.size/sr,0.01))
|
||||
flag=" <<< MÄNNLICH/Kollaps?" if (f0==f0 and f0<150) else ""
|
||||
print(f" [{i:2}] F0={f0:6.1f}Hz{flag}", flush=True)
|
||||
arr=np.array([x for x in f0s if x==x])
|
||||
print(f" -> F0 median {np.median(arr):.0f} min {arr.min():.0f} max {arr.max():.0f} | RTF~{np.median(rtfs):.2f}", flush=True)
|
||||
|
||||
run(0.9, None)
|
||||
run(0.9, 3.0)
|
||||
print("NC_DONE")
|
||||
@@ -1,57 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Warum 'manchmal zu laut' + 'Stimme verändert sich am Anfang'? Stream-Kopf-Logik N× laufen lassen
|
||||
und Gain, Crop-Punkt, Kopf-Segmente, Whisper-Start, rms/peak protokollieren."""
|
||||
import os, numpy as np, librosa, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
src,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
srt,_ = librosa.effects.trim(src, top_db=30); ref=os.path.join(BASE,"ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
TARGET_RMS=0.09; LEAD="Tja. "
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=0.9)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr=m.sample_rate
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
TEXTS = ["Hallo Commander, ich höre dich.",
|
||||
"Natürlich, Commander. Das Backup ist sauber durchgelaufen."]
|
||||
|
||||
def run_once(text):
|
||||
chunks=[]
|
||||
for c in m.generate_audio_stream(vs, LEAD+text, frames_after_eos=4):
|
||||
c=c.numpy() if hasattr(c,"numpy") else np.asarray(c)
|
||||
chunks.append(np.asarray(c,dtype=np.float32).reshape(-1))
|
||||
full=np.concatenate(chunks)
|
||||
# Kopf adaptiv (wie Server): bis >=2 Segmente oder 2.5s
|
||||
head=[]; hl=0; head_arr=None
|
||||
for c in chunks:
|
||||
head.append(c); hl+=c.size
|
||||
if hl < int(0.5*sr): continue
|
||||
a=np.concatenate(head)
|
||||
if len(librosa.effects.split(a, top_db=35))>=2 or hl>=int(2.5*sr):
|
||||
head_arr=a; break
|
||||
if head_arr is None: head_arr=np.concatenate(head)
|
||||
iv=librosa.effects.split(head_arr, top_db=35)
|
||||
segs=[(round(s/sr,2),round(e/sr,2)) for s,e in iv]
|
||||
# Gain aus voiced (aktuelle Server-Logik)
|
||||
voiced=np.concatenate([head_arr[s:e] for s,e in iv]) if len(iv) else head_arr
|
||||
rms=float(np.sqrt(np.mean(voiced**2))) or 1e-9
|
||||
gain=TARGET_RMS/rms
|
||||
# Crop (aktuell): kurz vor echtem Wort
|
||||
if len(iv)>=2:
|
||||
cut=max(iv[0][1], iv[1][0]-int(0.06*sr)); cropped=full[cut:]
|
||||
else:
|
||||
cropped=full
|
||||
out=np.clip(cropped*gain,-0.95,0.95)
|
||||
sf.write(os.path.join(BASE,"out_diag_s.wav"), out, sr)
|
||||
seg,_=w.transcribe(os.path.join(BASE,"out_diag_s.wav"), language="de", beam_size=5)
|
||||
start=(" ".join(x.text for x in seg)).strip()[:30]
|
||||
return dict(segs=segs, voiced_rms=round(rms,3), gain=round(gain,2),
|
||||
out_rms=round(float(np.sqrt(np.mean(out**2))),3), out_peak=round(float(np.abs(out).max()),3),
|
||||
start=start)
|
||||
|
||||
for text in TEXTS:
|
||||
print(f"=== {text[:30]!r} ===")
|
||||
for i in range(4):
|
||||
print(f" run{i}:", run_once(text))
|
||||
print("DIAG2_DONE")
|
||||
@@ -1,59 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Test der ROBUSTEN Stream-Kopf-Logik: fester 1.8s-Kopf + Crop am ersten SUBSTANZIELLEN Wort-Segment
|
||||
(ignoriert Mini-Blips) + FIXER Gain. Ziel: Start immer echtes Wort, out_rms stabil ~0.09, kein Clip."""
|
||||
import os, numpy as np, librosa, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
src,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
srt,_ = librosa.effects.trim(src, top_db=30); ref=os.path.join(BASE,"ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
TARGET_RMS=0.09; LEAD="Tja. "
|
||||
BASE_GAIN = TARGET_RMS / 0.18 # ~0.5 (raw full-rms ~0.18 konsistent)
|
||||
GMIN, GMAX = 0.7*BASE_GAIN, 1.4*BASE_GAIN # Gain-Clamp -> kann NIE explodieren
|
||||
HEAD_FIXED = int(2.0*24000)
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=0.9)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr=m.sample_rate
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
TEXTS = ["Hallo Commander, ich höre dich.",
|
||||
"Natürlich, Commander. Das Backup ist sauber durchgelaufen."]
|
||||
|
||||
def crop_and_gain(a):
|
||||
"""Crop am ersten SUBSTANZIELLEN Wort-Segment nach dem Lead (>=0.2s, ignoriert Mini-Blips);
|
||||
Gain aus voiced-rms, GECLAMPT (kann nie explodieren)."""
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
voiced = np.concatenate([a[s:e] for s,e in iv]) if len(iv) else a
|
||||
rms = float(np.sqrt(np.mean(voiced**2))) or 1e-9
|
||||
gain = float(np.clip(TARGET_RMS/rms, GMIN, GMAX))
|
||||
cut = 0
|
||||
if len(iv) >= 2:
|
||||
for k in range(1, len(iv)):
|
||||
if (iv[k][1]-iv[k][0]) >= int(0.20*sr):
|
||||
cut = max(iv[k-1][1], iv[k][0]-int(0.06*sr)); break
|
||||
return a[cut:], round(cut/sr,2), round(gain,2)
|
||||
|
||||
def run_once(text):
|
||||
head=[]; hl=0; head_done=False; head_arr=None; rest=[]
|
||||
for c in m.generate_audio_stream(vs, LEAD+text, frames_after_eos=4):
|
||||
c=c.numpy() if hasattr(c,"numpy") else np.asarray(c); c=np.asarray(c,dtype=np.float32).reshape(-1)
|
||||
if not head_done:
|
||||
head.append(c); hl+=c.size
|
||||
if hl>=HEAD_FIXED: head_arr=np.concatenate(head); head_done=True
|
||||
else: rest.append(c)
|
||||
if head_arr is None: head_arr=np.concatenate(head)
|
||||
cropped_head, cutpt, gain = crop_and_gain(head_arr)
|
||||
full = np.concatenate([cropped_head]+rest)
|
||||
out=np.clip(full*gain,-0.95,0.95)
|
||||
sf.write(os.path.join(BASE,"out_diag_s3.wav"), out, sr)
|
||||
seg,_=w.transcribe(os.path.join(BASE,"out_diag_s3.wav"), language="de", beam_size=5)
|
||||
start=(" ".join(x.text for x in seg)).strip()[:32]
|
||||
return dict(cut=cutpt, gain=gain, out_rms=round(float(np.sqrt(np.mean(out**2))),3),
|
||||
out_peak=round(float(np.abs(out).max()),3), start=start)
|
||||
|
||||
print(f"gain-clamp=[{round(GMIN,2)},{round(GMAX,2)}] head={HEAD_FIXED/24000}s")
|
||||
for text in TEXTS:
|
||||
print(f"=== {text[:28]!r} ===")
|
||||
for i in range(5):
|
||||
print(f" run{i}:", run_once(text))
|
||||
print("DIAG3_DONE")
|
||||
@@ -1,44 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Verifiziert die 'Stimme wurde männlich'-Hypothese: misst F0 (Grundfrequenz) vieler Generierungen
|
||||
bei temp 0.9 vs 0.7. Weiblich (Saber-Klon) ~180-240Hz, männlich-Drift (Default 'juergen') ~100-140Hz.
|
||||
Sucht Ausreißer = Stimm-Identitäts-Kollaps."""
|
||||
import os, numpy as np, librosa, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
src,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
srt,_ = librosa.effects.trim(src, top_db=30); ref=os.path.join(BASE,"ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
LEAD="Tja. "
|
||||
TEXTS = [
|
||||
"Hallo Commander, ich höre dich.",
|
||||
"Natürlich, Commander, das mache ich sofort.",
|
||||
"Die Hitze in Hamburg ist heute wirklich heftig.",
|
||||
"Klar, ich kümmere mich gleich darum für dich.",
|
||||
"Guten Morgen, Commander, alles läuft sauber.",
|
||||
"Das Backup ist durchgelaufen, keine Fehler.",
|
||||
]
|
||||
|
||||
def median_f0(a, sr):
|
||||
f0,_,_ = librosa.pyin(a, fmin=80, fmax=400, sr=sr, frame_length=1024)
|
||||
v = f0[~np.isnan(f0)]
|
||||
return float(np.median(v)) if v.size else float("nan")
|
||||
|
||||
def run(temp):
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=temp)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr=m.sample_rate
|
||||
print(f"=== temp={temp} ===", flush=True)
|
||||
f0s=[]
|
||||
for i,t in enumerate(TEXTS):
|
||||
a = m.generate_audio(vs, LEAD+t, frames_after_eos=4)
|
||||
a = a.numpy() if hasattr(a,"numpy") else np.asarray(a)
|
||||
a = np.asarray(a,dtype=np.float32).reshape(-1)
|
||||
f0 = median_f0(a, sr); f0s.append(f0)
|
||||
flag = " <<< MÄNNLICH?" if (f0==f0 and f0<150) else ""
|
||||
print(f" [{i}] F0={f0:6.1f}Hz{flag} ({t[:30]})", flush=True)
|
||||
arr=np.array([x for x in f0s if x==x])
|
||||
print(f" -> median {np.median(arr):.0f}Hz, min {arr.min():.0f}, max {arr.max():.0f}", flush=True)
|
||||
|
||||
run(0.9)
|
||||
run(0.7)
|
||||
print("F0_DONE")
|
||||
@@ -1,47 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ABSCHLUSS-TEST: beweist die Prompt-Wirkung. Dieselbe Lucy-Antwort im NEUEN Stil (kurze Sätze,
|
||||
Punkte) vs. ALTEM Stil (Komma-Kette). Rendert auf :8134 + Desktop, misst Pausen (Whisper)."""
|
||||
import os, io, json, urllib.request
|
||||
import numpy as np, soundfile as sf
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
SERVER = "http://127.0.0.1:8134/tts"
|
||||
OUT = os.path.join(os.path.expanduser("~"), "Desktop", "lucy_test"); os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
# Gleicher Inhalt, zwei Stile — so würde Lucy VORHER (Komma-Kette) und NACHHER (Prompt: kurze Sätze) antworten.
|
||||
PAIRS = [
|
||||
("A_backup",
|
||||
"Klar, Commander, das Backup ist heute Nacht durchgelaufen, es gab keine Fehler, und die Datenbank läuft stabil bei 92 Prozent.",
|
||||
"Klar, Commander. Das Backup lief heute Nacht durch. Keine Fehler. Die Datenbank ist stabil bei 92 Prozent."),
|
||||
("B_status",
|
||||
"Also, wenn ich das kurz zusammenfasse, die Verbindung war weg, dann lief das Update durch, danach musste ich neu starten, und jetzt sind alle Dienste wieder oben.",
|
||||
"Kurz zusammengefasst, Commander. Die Verbindung war weg. Das Update lief durch. Ich musste neu starten. Jetzt sind alle Dienste wieder oben."),
|
||||
]
|
||||
|
||||
def render(text):
|
||||
req = urllib.request.Request(SERVER, data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"})
|
||||
return urllib.request.urlopen(req, timeout=180).read()
|
||||
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
|
||||
def measure(tag, text):
|
||||
wav = render(text); path = os.path.join(OUT, tag + ".wav"); open(path, "wb").write(wav)
|
||||
a, sr = sf.read(io.BytesIO(wav)); a = np.asarray(a, dtype=np.float32).reshape(-1); dur = len(a)/sr
|
||||
segs, _ = w.transcribe(path, language="de", beam_size=5, word_timestamps=True)
|
||||
words = [x for s in segs for x in (s.words or [])]
|
||||
gaps = [words[i+1].start - words[i].end for i in range(len(words)-1)]
|
||||
lang = [(words[i].word.strip(), gaps[i]) for i in range(len(gaps)) if gaps[i] >= 0.30]
|
||||
total_pause = sum(g for g in gaps if g >= 0.15)
|
||||
return dur, len(lang), total_pause, lang
|
||||
|
||||
for name, alt, neu in PAIRS:
|
||||
d_a, n_a, tp_a, l_a = measure(f"{name}_ALT_komma", alt)
|
||||
d_n, n_n, tp_n, l_n = measure(f"{name}_NEU_kurz", neu)
|
||||
print(f"\n### {name}")
|
||||
print(f" ALT (Komma-Kette): {d_a:4.1f}s | {n_a} LANG-Pausen | Summe Pausen {tp_a*1000:4.0f}ms")
|
||||
print(f" LANG: " + (", ".join(f'„{ww}\"={gg*1000:.0f}ms' for ww, gg in l_a) or "keine"))
|
||||
print(f" NEU (kurze Sätze): {d_n:4.1f}s | {n_n} LANG-Pausen | Summe Pausen {tp_n*1000:4.0f}ms")
|
||||
print(f" LANG: " + (", ".join(f'„{ww}\"={gg*1000:.0f}ms' for ww, gg in l_n) or "keine"))
|
||||
print(f" -> {d_a-d_n:+.1f}s Dauer, {tp_a*1000-tp_n*1000:+.0f}ms Pausen-Summe")
|
||||
print("\n=== ABSCHLUSS-TEST FERTIG === (WAVs im Desktop\\lucy_test)")
|
||||
@@ -1,60 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""lucy_ready.py — Startklar-Check + finale Samples in EINEM Lauf.
|
||||
Bootet den ECHTEN Server (FastAPI-TestClient triggert lifespan: Modell-Load, A1-Cache, STATE),
|
||||
prüft /health, und rendert die finalen Hörproben über die REALEN Endpoints /tts und /tts/stream
|
||||
(exakt das, was die Lucy-App auf :8130 aufruft) — inkl. Live-TTFB der Streaming-Antwort.
|
||||
Legt die Samples auf den Desktop, schreibt lucy_ready_report.json.
|
||||
"""
|
||||
import os, time, json
|
||||
import numpy as np, soundfile as sf
|
||||
from fastapi.testclient import TestClient
|
||||
import pocket_server as ps
|
||||
|
||||
DESK = os.path.join(os.path.expanduser("~"), "Desktop", "lucy_samples")
|
||||
os.makedirs(DESK, exist_ok=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel": "Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die "
|
||||
"Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
LONG = ("Guten Morgen, Commander. "
|
||||
"Das nächtliche Backup ist sauber durchgelaufen und es gab keine Fehler. "
|
||||
"Der Dienst läuft stabil und die Engine antwortet zügig. "
|
||||
"Wenn du möchtest, fasse ich die offenen Punkte für heute zusammen.")
|
||||
|
||||
report = {}
|
||||
t0 = time.time()
|
||||
with TestClient(ps.app) as c: # <-- triggert lifespan (echter Boot)
|
||||
report["boot_s"] = round(time.time() - t0, 1)
|
||||
h = c.get("/health").json()
|
||||
report["health"] = h
|
||||
print(f"BOOT in {report['boot_s']}s health={h}")
|
||||
|
||||
for name, text in SENT.items(): # /tts (ganze Datei) -> Desktop
|
||||
t = time.time(); r = c.post("/tts", json={"text": text}); dt = time.time() - t
|
||||
assert r.status_code == 200, f"/tts {name} -> {r.status_code}"
|
||||
open(os.path.join(DESK, f"lucy_final_{name}.wav"), "wb").write(r.content)
|
||||
print(f"/tts {name:6}: {dt:4.2f}s audio={r.headers.get('X-Audio-Seconds')}s")
|
||||
|
||||
# /tts/stream: lange Antwort wie Lucy LIVE spricht -> TTFB messen + PCM zu WAV
|
||||
sr = int(h.get("sr") or 24000)
|
||||
t = time.time(); first = None; chunks = []
|
||||
with c.stream("POST", "/tts/stream", json={"text": LONG}) as s:
|
||||
sr = int(s.headers.get("X-Sample-Rate", sr))
|
||||
for ch in s.iter_bytes():
|
||||
if ch:
|
||||
if first is None:
|
||||
first = time.time() - t
|
||||
chunks.append(ch)
|
||||
a = np.frombuffer(b"".join(chunks), dtype="<i2").astype(np.float32) / 32767.0
|
||||
sf.write(os.path.join(DESK, "lucy_final_antwort.wav"), a, sr)
|
||||
total = time.time() - t
|
||||
report["stream_ttfb_s"] = round(first or 0, 2)
|
||||
report["stream_total_s"] = round(total, 2)
|
||||
report["stream_audio_s"] = round(len(a) / sr, 1)
|
||||
print(f"/tts/stream: TTFB={first:.2f}s total={total:.2f}s audio={len(a)/sr:.1f}s")
|
||||
|
||||
json.dump(report, open(os.path.join(ps.BASE, "lucy_ready_report.json"), "w"), indent=2)
|
||||
print("LUCY_READY" if h.get("status") == "ok" else "NOT_READY")
|
||||
@@ -1,64 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""make_samples.py — erzeugt Lucy-Hörproben auf dem Desktop, um A1 (safetensors-Cache) und
|
||||
A2 (Referenz-Cleaning) hörbar zu testen. Nutzt 1:1 die Produktionslogik aus pocket_server.py
|
||||
(cleanup + Doppel-Wächter _gen_gated), läuft offline mit dem lokal gecachten german_24l-Modell.
|
||||
|
||||
Output: %USERPROFILE%\\Desktop\\lucy_samples\\
|
||||
lucy_clean_{kurz,mittel,lang}.wav (A2 AN: Highpass+Peak-Norm, Stimme aus safetensors-Cache)
|
||||
lucy_raw_{kurz,mittel,lang}.wav (A2 AUS: Roh-Referenz, direkt geklont)
|
||||
So hörst du den Cleaning-Unterschied direkt im A/B.
|
||||
"""
|
||||
import os, time
|
||||
import numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
import pocket_server as ps # reuse: _prep_ref, cleanup, _gen_gated, _fingerprint, Konstanten
|
||||
|
||||
DESK = os.path.join(os.path.expanduser("~"), "Desktop", "lucy_samples")
|
||||
os.makedirs(DESK, exist_ok=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel": "Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die "
|
||||
"Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
|
||||
|
||||
def build(ref_clean: bool, tag: str, use_cache: bool):
|
||||
ps.REF_CLEAN = ref_clean # A2-Schalter zur Laufzeit setzen
|
||||
ref = ps._prep_ref() # schreibt ref.wav (ggf. gecleant)
|
||||
print(f"\n[{tag}] lade Modell {ps.LANG} (lsd={ps.LSD} temp={ps.TEMP} nc={ps.NOISE_CLAMP}) ...")
|
||||
t0 = time.time()
|
||||
m = TTSModel.load_model(language=ps.LANG, lsd_decode_steps=ps.LSD, temp=ps.TEMP,
|
||||
noise_clamp=ps.NOISE_CLAMP, quantize=ps.QUANTIZE)
|
||||
sr = m.sample_rate
|
||||
print(f"[{tag}] Modell in {time.time()-t0:.1f}s (sr={sr})")
|
||||
if use_cache: # A1: einmal exportieren, dann aus Cache laden
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
try:
|
||||
ps.export_model_state(vs, ps.VOICE_ST)
|
||||
vs = m.get_state_for_audio_prompt(ps.VOICE_ST)
|
||||
print(f"[{tag}] Voice-State exportiert+geladen <- {os.path.basename(ps.VOICE_ST)}")
|
||||
except Exception as e:
|
||||
print(f"[{tag}] Export/Cache fehlgeschlagen, nutze Live-Klon: {e}")
|
||||
else:
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
ref_audio, _ = librosa.load(ref, sr=sr, mono=True)
|
||||
ps.STATE.update(m=m, vs=vs, sr=sr, ref_fp=ps._fingerprint(ref_audio, sr))
|
||||
for name, text in SENT.items():
|
||||
t0 = time.time()
|
||||
a = ps.cleanup(ps._gen_gated(text), sr) # Produktions-Wächter + cleanup
|
||||
dt = time.time() - t0
|
||||
p = os.path.join(DESK, f"lucy_{tag}_{name}.wav")
|
||||
sf.write(p, a, sr)
|
||||
print(f"[{tag}] {name:6} gen={dt:5.2f}s audio={a.size/sr:4.1f}s -> {p}")
|
||||
ps.STATE.clear()
|
||||
del m
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=== A2 AN (Highpass+Norm) + A1 safetensors-Cache ===")
|
||||
build(ref_clean=True, tag="clean", use_cache=True)
|
||||
print("\n=== A2 AUS (Roh-Referenz, direkt geklont) ===")
|
||||
build(ref_clean=False, tag="raw", use_cache=False)
|
||||
print(f"\nFERTIG. Samples liegen in: {DESK}")
|
||||
@@ -1,38 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Fix-Verifikation: rendert die Problem-/Regressionssätze auf :8134, misst Pausen + IST (Whisper)."""
|
||||
import os, io, json, urllib.request
|
||||
import numpy as np, soundfile as sf
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
SERVER = "http://127.0.0.1:8134/tts"
|
||||
OUT = os.path.join(os.environ["TEMP"], "lucy_fix"); os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
TESTS = [
|
||||
("01_kurz", "Schön, dass du da bist, Commander."),
|
||||
("07_lang", "Guten Morgen, Commander. Das nächtliche Backup ist um 3 Uhr durchgelaufen, es gab keine Fehler, und die Datenbank läuft mit 92 Prozent Auslastung völlig stabil."),
|
||||
("08_sehr_lang","Also, Commander, wenn ich das kurz zusammenfassen darf: heute früh war die Verbindung weg, dann lief das Update durch, danach musste ich zweimal neu starten, und jetzt sind endlich alle Dienste oben, das Gedächtnis ist warm, und wir können ganz entspannt weitermachen."),
|
||||
("09_akronyme", "Die CPU der RX 9070 XT und die GPU laufen mit ROCm, sagt der PC über das LAN. Es ist 21 Uhr UTC."),
|
||||
]
|
||||
|
||||
def render(text):
|
||||
req = urllib.request.Request(SERVER, data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"})
|
||||
return urllib.request.urlopen(req, timeout=180).read()
|
||||
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
for name, text in TESTS:
|
||||
wav = render(text)
|
||||
path = os.path.join(OUT, name + ".wav"); open(path, "wb").write(wav)
|
||||
a, sr = sf.read(io.BytesIO(wav)); a = np.asarray(a, dtype=np.float32).reshape(-1); dur = len(a)/sr
|
||||
segs, _ = w.transcribe(path, language="de", beam_size=5, word_timestamps=True)
|
||||
words = [x for s in segs for x in (s.words or [])]
|
||||
got = " ".join(x.word.strip() for x in words)
|
||||
pauses = [(words[i].word.strip(), words[i+1].start - words[i].end)
|
||||
for i in range(len(words)-1) if (words[i+1].start - words[i].end) >= 0.25]
|
||||
print(f"\n### {name} {dur:.1f}s {len(words)} Wörter")
|
||||
print(f"IST: {got}")
|
||||
if pauses:
|
||||
print("PAUSEN>=250ms: " + ", ".join(f'„{ww}\"={gg*1000:.0f}ms' for ww, gg in sorted(pauses, key=lambda t:-t[1])[:6]))
|
||||
else:
|
||||
print("PAUSEN>=250ms: KEINE")
|
||||
print("\n=== FERTIG ===")
|
||||
@@ -1,45 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, time, numpy as np, soundfile as sf, librosa, torch, torchaudio
|
||||
def _ta_load(p,*a,**k):
|
||||
x,sr=sf.read(p,dtype="float32"); return torch.from_numpy((x.T if x.ndim>1 else x[None,:]).copy()),sr
|
||||
def _ta_save(p,t,sample_rate=24000,*a,**k):
|
||||
sr=k.get("sample_rate",sample_rate); x=t.detach().cpu().numpy()
|
||||
sf.write(p,(x.T if x.ndim>1 else x.reshape(-1)),int(sr))
|
||||
torchaudio.load=_ta_load; torchaudio.save=_ta_save
|
||||
def _wl(f,sr=16000):
|
||||
y,_=librosa.load(f,sr=sr,mono=True); return y.astype(np.float32)
|
||||
try:
|
||||
import whisper.audio as _wa; _wa.load_audio=_wl
|
||||
import whisper.transcribe as _wt; _wt.load_audio=_wl
|
||||
except Exception as e: print("whisper-patch:", e, flush=True)
|
||||
import outetts
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
ref = os.path.join(BASE, "ref.wav")
|
||||
y,_ = librosa.load(os.path.join(BASE,"ref.mp3"), sr=24000, mono=True)
|
||||
yt,_ = librosa.effects.trim(y, top_db=30)
|
||||
sf.write(ref, yt[:int(14*24000)], 24000)
|
||||
|
||||
cfg = outetts.ModelConfig(
|
||||
model_path=os.path.join(BASE,"models","OuteTTS-1.0-1B-Q8_0.gguf"),
|
||||
tokenizer_path="OuteAI/Llama-OuteTTS-1.0-1B",
|
||||
interface_version=outetts.InterfaceVersion.V3,
|
||||
backend=outetts.Backend.LLAMACPP,
|
||||
)
|
||||
cfg.n_gpu_layers = 999
|
||||
t0=time.time(); iface=outetts.Interface(config=cfg); print(f"[iface] {time.time()-t0:.1f}s", flush=True)
|
||||
spk=iface.create_speaker(ref); print("[clone ok]", flush=True)
|
||||
|
||||
OUT=os.path.join(BASE,"out"); os.makedirs(OUT,exist_ok=True)
|
||||
SENT={
|
||||
"kurz":"Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang":"Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
for name,text in SENT.items():
|
||||
for run in ("cold","warm"):
|
||||
t0=time.time(); out=iface.generate(config=outetts.GenerationConfig(text=text, speaker=spk)); dt=time.time()-t0
|
||||
p=os.path.join(OUT,f"{name}.wav"); out.save(p)
|
||||
x,sr=sf.read(p); secs=len(x)/sr
|
||||
print(f"[{name:6} {run:4}] gen={dt:5.2f}s audio={secs:5.2f}s RTF={dt/max(secs,0.01):.2f}", flush=True)
|
||||
print("PC_DONE", flush=True)
|
||||
@@ -1,471 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Lucy-Stimme: Pocket TTS (Kyutai) als lokaler CPU-Dienst. Klont Lucys Saber-Stimme, real-time, kein GPU."""
|
||||
import os, io, re, time, threading, logging
|
||||
import numpy as np, soundfile as sf, librosa
|
||||
from scipy.signal import butter, sosfilt
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import Response, JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from contextlib import asynccontextmanager
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pocket_tts import TTSModel, export_model_state
|
||||
from text_norm import normalize_numbers, fix_acronyms # Zahlen-Normalisierung + Akronym-Fix für die Stimme
|
||||
|
||||
log = logging.getLogger("lucy-tts"); logging.basicConfig(level=logging.INFO)
|
||||
BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
REF_MP3 = os.path.join(BASE, "ref.mp3")
|
||||
LANG = os.environ.get("LUCY_LANG", "german_24l")
|
||||
# Vom User per Ohr getunt (2026-06-28): lsd 10 + noise_clamp 2.5 + ref18 (Sweep-Sieger: sauberes
|
||||
# 'Commander', Timbre + logische Betonung, wenig Rauschen). temp 0.9 (lebendig). Per Env umschaltbar.
|
||||
LSD = int(os.environ.get("LUCY_LSD", "16")) # 16 statt 10: glattere Prosodie/Komma-Übergänge (User-A/B 2026-07-01), RTF bleibt <1
|
||||
TEMP = float(os.environ.get("LUCY_TEMP", "0.9"))
|
||||
def _parse_nc(v): # noise_clamp: Zahl, oder None bei "none"/leer/0
|
||||
return None if v.strip().lower() in ("", "none", "0") else float(v)
|
||||
NOISE_CLAMP = _parse_nc(os.environ.get("LUCY_NOISE_CLAMP", "2.5")) # dämpft Artefakte+Rauschen+Kollaps; 3.0 = rauschärmer
|
||||
FEOS = int(os.environ.get("LUCY_FRAMES_AFTER_EOS", "4")) # kurze Sätze bekamen sonst 0ms Schwanz
|
||||
TARGET_RMS = float(os.environ.get("LUCY_TARGET_RMS", "0.09")) # User: 0.09 u. 0.06 beide gut -> 0.09 Default
|
||||
LEAD = os.environ.get("LUCY_LEAD", "Tja. ") # Wegwerf-Lead-Wort -> natürlicher Onset fürs echte 1. Wort
|
||||
QUANTIZE = os.environ.get("LUCY_QUANTIZE", "1") not in ("0", "false", "False") # int8: ~27% schneller, lt. Doku ohne Qualitätsverlust
|
||||
PAD_S = float(os.environ.get("LUCY_PAD_S", "0.025")) # Satz-Polster (klein = flüssigere Übergänge bei langen Antworten)
|
||||
TAIL_DB = int(os.environ.get("LUCY_TAIL_DB", "30")) # Tail-Trim (dB unter Peak): kleiner = mehr Nachlauf-Stille weg = kürzere Satz-Pausen (gemessen: 45 ließ 500-680ms stehen)
|
||||
MAX_GAP_S = float(os.environ.get("LUCY_MAX_GAP", "0.20")) # interne Sprech-Pausen deckeln: Pockets Komma-Pausen ziehen bis 580ms -> hart auf 200ms kappen (flüssiger)
|
||||
F0_FLOOR = float(os.environ.get("LUCY_F0_FLOOR", "160")) # Hz: drunter = männlicher Kollaps (Klon~220, male~100)
|
||||
FP_FLOOR = float(os.environ.get("LUCY_FP_FLOOR", "0.94")) # Fingerabdruck-Ähnlichkeit (ohne MFCC0): drunter = Stimm-Drift (gut 0.956-0.98, drift 0.928)
|
||||
MAX_TRIES = int(os.environ.get("LUCY_MAX_TRIES", "3")) # max. Versuche gegen Stimm-Anomalien
|
||||
FP_MIN_S = float(os.environ.get("LUCY_FP_MIN_S", "2.0")) # B3: FP-Drift-Gate erst ab so viel stimmhafter Dauer (kurze Audios = verrauschter Fingerabdruck = Fehlalarm). F0-Gate bleibt immer aktiv.
|
||||
# A1: Voice-State einmalig nach safetensors exportieren -> Start lädt kvcache statt neu zu klonen.
|
||||
VOICE_ST = os.environ.get("LUCY_VOICE_ST", os.path.join(BASE, "lucy_voice.safetensors"))
|
||||
FORCE_RECLONE = os.environ.get("LUCY_FORCE_RECLONE", "0") not in ("0", "false", "False")
|
||||
# A2: Referenz-Cleaning (pocket reproduziert die Sample-Qualität mit) — Highpass + Peak-Norm, abschaltbar.
|
||||
REF_CLEAN = os.environ.get("LUCY_REF_CLEAN", "1") not in ("0", "false", "False")
|
||||
REF_HPF_HZ = float(os.environ.get("LUCY_REF_HPF_HZ", "70")) # Rumpel/Netzbrumm raus
|
||||
REF_SECONDS = float(os.environ.get("LUCY_REF_SECONDS", "18")) # Klon-Referenzlänge
|
||||
# B2: parallele Satz-Worker. SWEEP-ERGEBNIS (9700X, german_24l, 30.06.): seriell hat die BESTE
|
||||
# TTFB (3.6s vs 6–8s Pool) UND RTF 0.74<1 (generiert schneller als Echtzeit -> Streaming spielt
|
||||
# lückenlos). Daher Default 0 = seriell für Lucys Live-Stimme. Der Pool (>=1) lohnt NUR für
|
||||
# Batch-/tts (ganze Datei am Stück): Durchsatz-Sweet-Spot 4w×2t (RTF 0.44) bzw. 3w×2t (RTF 0.50).
|
||||
# Threads pro Worker via LUCY_WORKER_THREADS.
|
||||
WORKERS = int(os.environ.get("LUCY_WORKERS", "0"))
|
||||
# TTFB-Opt „kurzer erster Chunk": erster Stream-Chunk bleibt kurz -> Lucy spricht früher.
|
||||
# MIT B3 (FP-Gate überspringt kurze Audios) GEMESSEN STARK: TTFB 1.31s statt 3.74s (30.06.),
|
||||
# Wall ~gleich, RTF<1 (lückenlos). Daher Default AN. (Ohne B3 wäre es schlechter -> beides gehört
|
||||
# zusammen.) Nur Stream-Pfad.
|
||||
FAST_FIRST = os.environ.get("LUCY_FAST_FIRST", "1") not in ("0", "false", "False")
|
||||
MIN_LEN = int(os.environ.get("LUCY_MIN_LEN", "55")) # Chunk-Bündelung: größer = weniger Übergänge = flüssiger
|
||||
# Laufzeit-Regler (POST /tune, OHNE Neustart) -> Prosodie live nach Ohr A/B-testen.
|
||||
# temp/lsd/noise_clamp sind Modell-Load-Params und NICHT hier drin (die brauchen einen Neustart).
|
||||
TUNE = {"min_len": MIN_LEN, "pad_s": PAD_S, "fast_first": FAST_FIRST}
|
||||
# Umlaut-Fix: LLM (Hermes/Qwen) gibt manchmal ASCII-Umlaute aus (ueber/schoen/maerz) -> pocket liest
|
||||
# „ue" wörtlich. Wir wandeln NUR bekannte Ganzwörter zurück (sicher: „neue/aktuell/Steuer" bleiben).
|
||||
UMLAUT_FIX = os.environ.get("LUCY_UMLAUT_FIX", "1") not in ("0", "false", "False")
|
||||
STATE, LOCK = {}, threading.Lock()
|
||||
|
||||
# Bekannte deutsche Umlaut-Wörter (korrekt geschrieben). Die ASCII-Schlüssel (ä->ae, ö->oe, ü->ue,
|
||||
# ß->ss) werden daraus AUTOMATISCH abgeleitet -> wenig Fehlerquelle. Erweiterbar via LUCY_UMLAUT_EXTRA.
|
||||
_UML_WORDS = (
|
||||
"über überall übrigens übernehmen überzeugt überprüfen für fürs fünf fünfzehn fünfzig müssen "
|
||||
"müsste müssten können könnt könnte könnten könig königin möchte möchten möchtest möglich "
|
||||
"möglichst möglichkeit würde würden würdest müde prüfen prüft prüfung zurück natürlich gemütlich "
|
||||
"grün grüne grüße grüßen drücken dürfen darüber gegenüber dafür wofür hierfür führen führt "
|
||||
"führung fühlen gefühl gefühle tür türen glück glücklich stück stücke brücke küche kühl "
|
||||
"kühlschrank früh früher frühstück frühling bücher büro bürger südlich süden schüler schützen "
|
||||
"wünschen wünsche übung übungen künstler künstlich rücken rückkehr brüder lücke müll gründe "
|
||||
"gründen begründung vergnügen "
|
||||
"schön schöne schöner schönste schönheit möbel höher höhe hören gehört gehören größe größer "
|
||||
"größte öffnen öffnet geöffnet öffentlich öl östlich löschen löffel lösung lösen börse dörfer "
|
||||
"wörter wörterbuch völlig völker störung stören zwölf böse höflich öfter vögel mögen "
|
||||
"ähnlich änderung ändern ärger ärgern ärztin äußern äußerst gespräch gespräche hängen länger "
|
||||
"länge mädchen männer märz nächste nächsten nähe näher qualität universität städte tätigkeit "
|
||||
"täglich träume träumen väter wäre wären während wärme zähne erklären erklärung verändern "
|
||||
"gefährlich geschäft geschäfte jährlich käse hände kälte plätze sätze wälder fähig fähigkeit"
|
||||
).split()
|
||||
|
||||
def _build_umlaut_map():
|
||||
words = list(_UML_WORDS)
|
||||
extra = os.environ.get("LUCY_UMLAUT_EXTRA", "")
|
||||
words += [w.strip() for w in extra.replace(",", " ").split() if w.strip()]
|
||||
# gängige Flexionsendungen mitnehmen -> deckt mögliche/möglichen/größeren/schönes... ab.
|
||||
# Bogus-Formen (z.B. „möchtee") sind harmlos: sie tauchen in echtem Text nie auf.
|
||||
endings = ("", "e", "en", "er", "es", "em", "n", "s")
|
||||
m = {}
|
||||
for w in words:
|
||||
base = w.lower()
|
||||
for suf in endings:
|
||||
wl = base + suf
|
||||
ascii_w = wl.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss")
|
||||
if ascii_w != wl: # nur echte Umlaut-Wörter
|
||||
m.setdefault(ascii_w, wl)
|
||||
return m
|
||||
|
||||
_UML_MAP = _build_umlaut_map()
|
||||
_UML_RX = re.compile(r"\b(" + "|".join(sorted((re.escape(k) for k in _UML_MAP), key=len, reverse=True))
|
||||
+ r")\b", re.IGNORECASE) if _UML_MAP else None
|
||||
|
||||
def _fix_umlauts(text: str) -> str:
|
||||
"""Wandelt NUR bekannte ASCII-Umlaut-Ganzwörter zurück (ueber->über), case-erhaltend."""
|
||||
if not UMLAUT_FIX or not _UML_RX:
|
||||
return text
|
||||
def _repl(mo):
|
||||
s = mo.group(0); u = _UML_MAP[s.lower()]
|
||||
return u[:1].upper() + u[1:] if s[:1].isupper() else u
|
||||
return _UML_RX.sub(_repl, text)
|
||||
|
||||
def _prep_ref():
|
||||
"""A2: Klon-Referenz aufbereiten. pocket-tts reproduziert die Sample-Qualität mit, daher optional
|
||||
Highpass (Rumpeln/Netzbrumm) + Peak-Normalisierung. Per LUCY_REF_CLEAN=0 abschaltbar (A/B per Ohr)."""
|
||||
ref = os.path.join(BASE, "ref.wav")
|
||||
y, _ = librosa.load(REF_MP3, sr=24000, mono=True)
|
||||
y, _ = librosa.effects.trim(y, top_db=30)
|
||||
if REF_CLEAN:
|
||||
sos = butter(4, REF_HPF_HZ, btype="highpass", fs=24000, output="sos")
|
||||
y = sosfilt(sos, y).astype(np.float32)
|
||||
# nach dem Highpass nochmal randstille trimmen, dann Peak-Norm gegen zu leise/zu laute Referenz
|
||||
y, _ = librosa.effects.trim(y, top_db=30)
|
||||
peak = float(np.max(np.abs(y))) or 1.0
|
||||
y = (y * (0.95 / peak)).astype(np.float32)
|
||||
sf.write(ref, y[: int(REF_SECONDS * 24000)], 24000)
|
||||
return ref
|
||||
|
||||
def _drop_tail_blip(a: np.ndarray, sr: int) -> np.ndarray:
|
||||
"""Entfernt das End-'taa': isolierter, sehr leiser + kurzer Schwanz-Blip (Modell-Halluzination,
|
||||
intermittierend bei temp 0.9). Verifiziert: trifft Blip (rms-ratio 0.15), schont echte Kurzwörter
|
||||
wie 'Fehler' (ratio 0.55). Bedingung ALLE: große Lücke + viel leiser als Sprache + kurz."""
|
||||
for _ in range(3): # bis zu 3 Blips hintereinander
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) < 2:
|
||||
break
|
||||
speech_rms = float(np.median([np.sqrt(np.mean(a[s:e] ** 2)) for s, e in iv[:-1]]))
|
||||
s, e = iv[-1]
|
||||
last_dur = (e - s) / sr
|
||||
last_rms = float(np.sqrt(np.mean(a[s:e] ** 2)))
|
||||
gap = (s - iv[-2][1]) / sr
|
||||
if gap > 0.35 and last_rms < 0.30 * speech_rms and last_dur < 0.35:
|
||||
a = a[: iv[-2][1]]
|
||||
else:
|
||||
break
|
||||
return a
|
||||
|
||||
def _crop_lead(a: np.ndarray, sr: int) -> np.ndarray:
|
||||
"""Schneidet das Wegwerf-Lead-Wort weg -> echtes 1. Wort behält seinen vollen, natürlichen Onset
|
||||
(pocket-tts startet sonst mid-Phonem = 'vorne abgeschnitten'). Schnitt KURZ VOR dem echten Wort
|
||||
(nicht direkt hinter dem Lead), damit auch die variable, teils lange Pause nach 'Tja.' verschwindet."""
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) >= 2:
|
||||
cut = max(iv[0][1], iv[1][0] - int(0.06 * sr)) # 60ms vor echtem Wort, aber hinter dem Lead
|
||||
a = a[cut:]
|
||||
return a
|
||||
|
||||
def _compress_gaps(a: np.ndarray, sr: int, max_gap: float = MAX_GAP_S, top_db: int = 30) -> np.ndarray:
|
||||
"""Deckelt INTERNE Sprech-Pausen (Pockets Komma-Pausen ziehen gemessen bis 580ms) auf max_gap.
|
||||
Findet stimmhafte Segmente, fügt sie mit gekappter Lücke wieder zusammen -> flüssiger, ohne die
|
||||
Sprache selbst anzutasten (nur Stille wird gekürzt). Onset-Vorlauf + Schwanz bleiben unberührt."""
|
||||
if a.size == 0 or max_gap <= 0:
|
||||
return a
|
||||
iv = librosa.effects.split(a, top_db=top_db)
|
||||
if len(iv) < 2:
|
||||
return a
|
||||
cap = int(max_gap * sr)
|
||||
out = [a[: iv[0][1]]] # Kopf inkl. natürlichem Onset-Vorlauf
|
||||
for k in range(1, len(iv)):
|
||||
g0 = iv[k - 1][1]
|
||||
keep = min(iv[k][0] - g0, cap)
|
||||
if keep > 0:
|
||||
out.append(a[g0 : g0 + keep]) # gekappte Pause (Wort-Ausklang bleibt erhalten)
|
||||
out.append(a[iv[k][0] : iv[k][1]]) # nächstes stimmhaftes Segment
|
||||
out.append(a[iv[-1][1] :]) # Schwanz (bereits getrimmt)
|
||||
return np.concatenate(out)
|
||||
|
||||
def cleanup(a: np.ndarray, sr: int) -> np.ndarray:
|
||||
"""v4 (2026-06-28): Onset-Fix + RMS-Lautstärke + Blip-Killer.
|
||||
- _drop_tail_blip gegen End-'taa'
|
||||
- _crop_lead entfernt Wegwerf-Lead -> voller Onset vorne (kein Abschneiden mehr)
|
||||
- nur HINTEN trimmen (vorne unangetastet), RMS-Normalisierung auf TARGET_RMS (statt lautem Peak 0.95)
|
||||
- 8ms-Fades + 80ms-Atempause vorne+hinten."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if a.size == 0: return a
|
||||
a = _drop_tail_blip(a, sr)
|
||||
a = _crop_lead(a, sr)
|
||||
# nur den Schwanz trimmen (vorderen Onset behalten)
|
||||
rev, _ = librosa.effects.trim(a[::-1], top_db=TAIL_DB)
|
||||
a = rev[::-1] if rev.size else a
|
||||
a = _compress_gaps(a, sr) # interne Komma-Pausen deckeln (größter Glättungs-Gewinn bei langen Sätzen)
|
||||
# RMS-Normalisierung auf Zielpegel (gegen 'zu laut') + Peak-Sicherheits-Clamp
|
||||
rms = float(np.sqrt(np.mean(a ** 2))) or 1e-9
|
||||
a = a * (TARGET_RMS / rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > 0.9: a = a * (0.9 / peak)
|
||||
fi = min(int(0.008 * sr), a.size // 2) # 8ms Fade gegen Klicks
|
||||
if fi > 0:
|
||||
a[:fi] *= np.linspace(0.0, 1.0, fi, dtype=np.float32)
|
||||
a[-fi:] *= np.linspace(1.0, 0.0, fi, dtype=np.float32)
|
||||
pad = np.zeros(int(TUNE["pad_s"] * sr), dtype=np.float32) # Atempause/Anti-Klick (klein -> flüssiger Satz-Übergang)
|
||||
return np.concatenate([pad, a, pad])
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
t0 = time.time()
|
||||
ref = _prep_ref() # ref.wav immer erzeugen -> Worker + Fingerabdruck
|
||||
need_export = FORCE_RECLONE or not os.path.exists(VOICE_ST) or \
|
||||
os.path.getmtime(VOICE_ST) < os.path.getmtime(REF_MP3)
|
||||
if WORKERS >= 1:
|
||||
# B2-Pool: safetensors einmalig erzeugen (kurz ein Modell), dann RAM freigeben; Worker
|
||||
# laden ihre eigene Instanz aus dem Cache. Hauptprozess hält danach KEIN Modell.
|
||||
log.info("Starte Worker-Pool (%d) — pocket-tts %s lsd=%d temp=%.2f nc=%s quantize=%s",
|
||||
WORKERS, LANG, LSD, TEMP, NOISE_CLAMP, QUANTIZE)
|
||||
if need_export:
|
||||
log.info("Erzeuge Voice-Cache %s ...", os.path.basename(VOICE_ST))
|
||||
mtmp = TTSModel.load_model(language=LANG, lsd_decode_steps=LSD, temp=TEMP,
|
||||
noise_clamp=NOISE_CLAMP, quantize=QUANTIZE)
|
||||
try:
|
||||
export_model_state(mtmp.get_state_for_audio_prompt(ref), VOICE_ST)
|
||||
except Exception as e:
|
||||
log.warning("Voice-Export fehlgeschlagen (Worker klonen selbst): %s", e)
|
||||
sr0 = mtmp.sample_rate
|
||||
del mtmp
|
||||
else:
|
||||
sr0 = 24000
|
||||
ref_audio, _ = librosa.load(ref, sr=sr0, mono=True)
|
||||
STATE.update(sr=sr0, ref_fp=_fingerprint(ref_audio, sr0))
|
||||
pool = ProcessPoolExecutor(max_workers=WORKERS, initializer=_worker_init)
|
||||
list(pool.map(_warmup, range(WORKERS))) # alle Worker vorab hochfahren
|
||||
STATE["pool"] = pool
|
||||
log.info("Worker-Pool bereit (%d Prozesse) in %.1fs", WORKERS, time.time() - t0)
|
||||
else:
|
||||
log.info("Seriell — pocket-tts %s lsd=%d temp=%.2f nc=%s quantize=%s",
|
||||
LANG, LSD, TEMP, NOISE_CLAMP, QUANTIZE)
|
||||
m = TTSModel.load_model(language=LANG, lsd_decode_steps=LSD, temp=TEMP,
|
||||
noise_clamp=NOISE_CLAMP, quantize=QUANTIZE)
|
||||
if need_export:
|
||||
log.info("Klone Stimme aus Referenz -> Export %s", os.path.basename(VOICE_ST))
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
try:
|
||||
export_model_state(vs, VOICE_ST); log.info("Voice-State exportiert")
|
||||
except Exception as e:
|
||||
log.warning("Voice-Export fehlgeschlagen (nutze Live-Klon): %s", e)
|
||||
else:
|
||||
log.info("Lade gecachten Voice-State <- %s", os.path.basename(VOICE_ST))
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(VOICE_ST)
|
||||
except Exception as e:
|
||||
log.warning("Cache-Load fehlgeschlagen, klone neu: %s", e)
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
ref_audio, _ = librosa.load(ref, sr=m.sample_rate, mono=True)
|
||||
STATE.update(m=m, vs=vs, sr=m.sample_rate, ref_fp=_fingerprint(ref_audio, m.sample_rate))
|
||||
log.info("Lucy-Stimme bereit in %.1fs (sr=%d)", time.time() - t0, m.sample_rate)
|
||||
yield
|
||||
pool = STATE.get("pool")
|
||||
if pool is not None:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
STATE.clear()
|
||||
|
||||
app = FastAPI(title="Lucy TTS (Pocket)", lifespan=lifespan)
|
||||
|
||||
class Req(BaseModel):
|
||||
text: str
|
||||
|
||||
class PerfIn(BaseModel):
|
||||
msg: str = ""
|
||||
|
||||
class TuneIn(BaseModel):
|
||||
min_len: int | None = None # Chunk-Bündelung (größer = flüssiger, langsamere TTFB später)
|
||||
pad_s: float | None = None # Pause zwischen Stücken (Sekunden)
|
||||
fast_first: bool | None = None # erster Satz kurz halten (schnelle TTFB) an/aus
|
||||
|
||||
def _ready() -> bool:
|
||||
return STATE.get("pool") is not None or "m" in STATE
|
||||
|
||||
def _gen_sentences_ordered(sentences):
|
||||
"""Liefert je Satz fertiges float32-Audio IN REIHENFOLGE. Mit Worker-Pool laufen alle Sätze
|
||||
parallel (bis WORKERS gleichzeitig), werden aber in Eingabereihenfolge ausgegeben -> niedrige
|
||||
TTFB + korrekte Reihenfolge. Ohne Pool: seriell im Hauptprozess (wie bisher, unter LOCK)."""
|
||||
pool = STATE.get("pool")
|
||||
sr = STATE["sr"]
|
||||
if pool is not None:
|
||||
for fut in [pool.submit(_worker_gen, s) for s in sentences]:
|
||||
yield fut.result()
|
||||
else:
|
||||
with LOCK:
|
||||
for s in sentences:
|
||||
yield cleanup(_gen_gated_core(STATE, s), sr)
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok" if _ready() else "loading", "engine": "pocket-tts", "lang": LANG,
|
||||
"sr": STATE.get("sr"), "workers": WORKERS, "device": "cpu"}
|
||||
|
||||
@app.post("/perf")
|
||||
def perf(body: PerfIn):
|
||||
"""Client-Perf-Zeilen in DIESES Terminal loggen (der Nutzer hat den TTS-Log eh offen) ->
|
||||
„Lucy denkt lange"-Diagnose ohne Browser-DevTools."""
|
||||
log.info("[lucy-perf] %s", body.msg)
|
||||
return {"ok": True}
|
||||
|
||||
@app.get("/tune")
|
||||
def tune_get():
|
||||
"""Aktuelle Laufzeit-Regler. temp/lsd/nc stehen separat (Modell-Load, Neustart nötig)."""
|
||||
return {**TUNE, "note": "temp/lsd/noise_clamp brauchen Neustart (Env LUCY_TEMP/LSD/NOISE_CLAMP)"}
|
||||
|
||||
@app.post("/tune")
|
||||
def tune_set(body: TuneIn):
|
||||
"""Prosodie live ändern OHNE Neustart -> nächste Äußerung nutzt die neuen Werte."""
|
||||
if body.min_len is not None: TUNE["min_len"] = max(1, int(body.min_len))
|
||||
if body.pad_s is not None: TUNE["pad_s"] = max(0.0, float(body.pad_s))
|
||||
if body.fast_first is not None: TUNE["fast_first"] = bool(body.fast_first)
|
||||
log.info("[tune] %s", TUNE)
|
||||
return TUNE
|
||||
|
||||
@app.post("/tts")
|
||||
def tts(req: Req):
|
||||
if not _ready():
|
||||
return JSONResponse({"error": "loading"}, status_code=503)
|
||||
t0 = time.time(); sr = STATE["sr"]
|
||||
parts = list(_gen_sentences_ordered(_split_sentences(fix_acronyms(normalize_numbers(_fix_umlauts(req.text))), min_len=TUNE["min_len"])))
|
||||
a = np.concatenate(parts) if parts else np.zeros(0, np.float32)
|
||||
buf = io.BytesIO(); sf.write(buf, a, sr, format="WAV", subtype="PCM_16"); buf.seek(0)
|
||||
dur = a.size / sr; gen = time.time() - t0
|
||||
log.info("/tts %dZ audio=%.1fs gen=%.1fs rtf=%.2f", len(req.text), dur, gen, gen / max(dur, 0.01))
|
||||
return Response(buf.read(), media_type="audio/wav",
|
||||
headers={"X-Audio-Seconds": f"{dur:.2f}", "X-Gen-Seconds": f"{gen:.2f}"})
|
||||
|
||||
def _voiced_f0(a: np.ndarray, sr: int) -> float:
|
||||
"""Schnelle Grundfrequenz-Schätzung auf dem längsten stimmhaften Segment (max 0.6s) via yin.
|
||||
Für das Stimm-Kollaps-Gate: Klon ~220Hz, männlicher Default ~100Hz."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if not len(iv):
|
||||
return float("nan")
|
||||
s, e = max(iv, key=lambda x: x[1] - x[0])
|
||||
seg = a[s:min(e, s + int(0.6 * sr))]
|
||||
if seg.size < int(0.1 * sr):
|
||||
return float("nan")
|
||||
fv = librosa.yin(seg, fmin=80, fmax=400, sr=sr, frame_length=1024)
|
||||
return float(np.median(fv)) if fv.size else float("nan")
|
||||
|
||||
def _fingerprint(a: np.ndarray, sr: int) -> np.ndarray:
|
||||
"""Stimm-Fingerabdruck (MFCC mean+std über stimmhafte Frames, normiert). Für den Drift-Wächter:
|
||||
erkennt JEDEN Stimm-Wechsel (auch weiblich->andere weiblich, das der F0-Wächter durchließ)."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
iv = librosa.effects.split(a, top_db=30)
|
||||
if len(iv):
|
||||
a = np.concatenate([a[s:e] for s, e in iv])
|
||||
if a.size < int(0.2 * sr):
|
||||
return None
|
||||
m = librosa.feature.mfcc(y=a, sr=sr, n_mfcc=20)[1:] # MFCC0 (Energie) weglassen -> amplituden-invariant
|
||||
v = np.concatenate([m.mean(1), m.std(1)])
|
||||
return (v / (np.linalg.norm(v) + 1e-9)).astype(np.float32)
|
||||
|
||||
def _gen_gated_core(st: dict, text: str) -> np.ndarray:
|
||||
"""Einen Satz erzeugen mit DOPPEL-Wächter (best-of-N): männlicher Kollaps (F0<Floor) UND Stimm-Drift
|
||||
(Fingerabdruck-Ähnlichkeit zur Referenz < Floor) -> neu generieren; am Ende den ref-ähnlichsten,
|
||||
nicht-männlichen Kandidaten behalten. `st` = Zustand (STATE im Hauptprozess, _W im Worker)."""
|
||||
best_a, best_score = None, -2.0
|
||||
m, vs, sr, ref_fp = st["m"], st["vs"], st["sr"], st.get("ref_fp")
|
||||
for attempt in range(MAX_TRIES):
|
||||
audio = m.generate_audio(vs, LEAD + text, frames_after_eos=FEOS)
|
||||
a = audio.numpy() if hasattr(audio, "numpy") else np.asarray(audio)
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
f0 = _voiced_f0(a, sr)
|
||||
male = f0 == f0 and f0 < F0_FLOOR
|
||||
cropped = _crop_lead(a, sr) # ohne Lead -> vergleichbar mit Referenz
|
||||
# B3: FP-Drift nur bei genug stimmhafter Dauer prüfen (kurze Audios -> Fingerabdruck unzuverlässig -> Fehlalarm)
|
||||
iv = librosa.effects.split(cropped, top_db=30)
|
||||
voiced_s = (sum(int(e - s) for s, e in iv) / sr) if len(iv) else 0.0
|
||||
fp = _fingerprint(cropped, sr) if voiced_s >= FP_MIN_S else None
|
||||
sim = float(np.dot(ref_fp, fp)) if (ref_fp is not None and fp is not None) else 1.0
|
||||
score = sim - (1.0 if male else 0.0) # männlich hart abstrafen
|
||||
if score > best_score:
|
||||
best_score, best_a = score, a
|
||||
if not male and sim >= FP_FLOOR: # gut genug -> stop
|
||||
break
|
||||
log.warning("Stimm-Anomalie (F0=%.0f male=%s sim=%.3f<%.2f) -> regeneriere (try %d)",
|
||||
f0, male, sim, FP_FLOOR, attempt + 1)
|
||||
return best_a
|
||||
|
||||
def _gen_gated(text: str) -> np.ndarray:
|
||||
"""Serieller Hauptprozess-Pfad (nutzt STATE). Wird auch von make_samples.py verwendet."""
|
||||
return _gen_gated_core(STATE, text)
|
||||
|
||||
# --- B2: persistenter Worker-Pool (je Prozess eigene Modellinstanz + Voice-State aus A1-Cache) -----
|
||||
_W: dict = {} # Per-Prozess-Zustand des Workers
|
||||
|
||||
def _worker_init():
|
||||
"""Einmal pro Worker-Prozess: Torch-Threads pinnen (gegen Oversubscription), Modell laden,
|
||||
Voice-State aus dem safetensors-Cache (A1) ziehen (Fallback: live klonen)."""
|
||||
try:
|
||||
import torch
|
||||
torch.set_num_threads(int(os.environ.get("LUCY_WORKER_THREADS", "2")))
|
||||
except Exception:
|
||||
pass
|
||||
m = TTSModel.load_model(language=LANG, lsd_decode_steps=LSD, temp=TEMP,
|
||||
noise_clamp=NOISE_CLAMP, quantize=QUANTIZE)
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(VOICE_ST)
|
||||
except Exception:
|
||||
vs = m.get_state_for_audio_prompt(_prep_ref())
|
||||
ref_audio, _ = librosa.load(os.path.join(BASE, "ref.wav"), sr=m.sample_rate, mono=True)
|
||||
_W.update(m=m, vs=vs, sr=m.sample_rate, ref_fp=_fingerprint(ref_audio, m.sample_rate))
|
||||
|
||||
def _worker_gen(text: str) -> np.ndarray:
|
||||
"""Im Worker: Satz mit Doppel-Wächter erzeugen + cleanup. Rückgabe = fertiges float32-PCM (picklebar)."""
|
||||
return np.asarray(cleanup(_gen_gated_core(_W, text), _W["sr"]), dtype=np.float32)
|
||||
|
||||
def _warmup(_):
|
||||
if "m" not in _W:
|
||||
_worker_init()
|
||||
return _W["sr"]
|
||||
|
||||
_SENT_RX = re.compile(r".+?(?:[.!?…]+(?:\s|$)|$)", re.S)
|
||||
|
||||
def _split_sentences(text: str, min_len: int = 55, keep_first_short: bool = False) -> list[str]:
|
||||
"""Text in Sätze zerlegen und sehr kurze Teile bündeln. Für satzweise Generierung +
|
||||
F0-Gate pro Satz -> ein Kollaps in der Mitte langer Antworten erreicht den Nutzer NIE.
|
||||
keep_first_short=True: der ERSTE Teil bleibt eigenständig (auch wenn kurz) -> schnellste TTFB
|
||||
im Stream (Lucy fängt früher an zu sprechen), Rest wird normal gebündelt."""
|
||||
parts = [m.group(0).strip() for m in _SENT_RX.finditer(text.strip())]
|
||||
out: list[str] = []
|
||||
for p in parts:
|
||||
if not p:
|
||||
continue
|
||||
if not out:
|
||||
out.append(p) # erster Teil
|
||||
elif keep_first_short and len(out) == 1:
|
||||
out.append(p) # zweiter Teil startet frisch -> Index 0 bleibt kurz
|
||||
elif len(out[-1]) < min_len:
|
||||
out[-1] = f"{out[-1]} {p}"
|
||||
else:
|
||||
out.append(p)
|
||||
return out or [text.strip()]
|
||||
|
||||
def _to_pcm16(a: np.ndarray, gain: float) -> bytes:
|
||||
"""float -> 16-bit-PCM (LE), mit Gain + Sicherheits-Clip."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1) * gain
|
||||
np.clip(a, -0.95, 0.95, out=a)
|
||||
return (a * 32767.0).astype("<i2").tobytes()
|
||||
|
||||
@app.post("/tts/stream")
|
||||
def tts_stream(req: Req):
|
||||
"""Streamt rohes PCM16-mono (sr via Header X-Sample-Rate) für niedrige Time-to-first-audio.
|
||||
SATZWEISE Generierung mit F0-Gate PRO Satz: jeder Satz wird voll erzeugt, auf Stimm-Kollaps
|
||||
(männlich/F0<Floor) geprüft und bei Bedarf neu generiert, BEVOR er emittiert wird. So erreicht
|
||||
kein kollabiertes Audio den Nutzer — auch nicht mitten in langen Antworten (war die 'gruselige'
|
||||
Schwäche, da pocket lange Texte intern chunkt und einzelne Chunks kippen können).
|
||||
Jeder Satz läuft durch die bewährte cleanup()-Pipeline (Lead-Crop, Tail-Blip, RMS-Norm, Pads).
|
||||
B2: Mit Worker-Pool laufen die Sätze PARALLEL, werden aber in Reihenfolge emittiert (TTFB =
|
||||
erster Satz, restliche rechnen schon nebenher)."""
|
||||
if not _ready():
|
||||
return JSONResponse({"error": "loading"}, status_code=503)
|
||||
sr = STATE["sr"]
|
||||
sentences = _split_sentences(normalize_numbers(_fix_umlauts(req.text)), min_len=TUNE["min_len"], keep_first_short=TUNE["fast_first"]) # +Umlaut +Zahlen, tunebar
|
||||
|
||||
def pcm_stream():
|
||||
t0 = time.time(); total = 0; first = True
|
||||
for a in _gen_sentences_ordered(sentences): # parallel (Pool) bzw. seriell, immer in Reihenfolge
|
||||
total += a.size
|
||||
if first:
|
||||
log.info("/tts/stream TTFB=%.2fs (%d Sätze, workers=%d)", time.time() - t0, len(sentences), WORKERS)
|
||||
first = False
|
||||
yield _to_pcm16(a, 1.0) # cleanup hat schon auf TARGET_RMS normalisiert
|
||||
log.info("/tts/stream %dZ audio=%.1fs gen=%.1fs", len(req.text), total / sr, time.time() - t0)
|
||||
|
||||
return StreamingResponse(pcm_stream(), media_type="application/octet-stream",
|
||||
headers={"X-Sample-Rate": str(sr)})
|
||||
@@ -1,47 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, time, glob, numpy as np, soundfile as sf
|
||||
from pocket_tts import TTSModel
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = os.path.join(BASE, "out_preset"); os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
t0 = time.time()
|
||||
m = TTSModel.load_model(language="german_24l")
|
||||
print(f"[load] {time.time()-t0:.1f}s sr={m.sample_rate}", flush=True)
|
||||
|
||||
voice = None
|
||||
try:
|
||||
from pocket_tts.default_parameters import get_default_voice_for_language
|
||||
voice = get_default_voice_for_language(str(getattr(m, "origin", "german_24l")))
|
||||
except Exception as e:
|
||||
print("default-voice fehler:", e, flush=True)
|
||||
for cand in [voice, "anna", "vera", "juergen", "cosette"]:
|
||||
if not cand: continue
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(cand); voice = cand; break
|
||||
except Exception as e:
|
||||
print(f"voice {cand} nicht nutzbar: {str(e)[:60]}", flush=True)
|
||||
print(f"[voice] {voice}", flush=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
for name, text in SENT.items():
|
||||
for run in ("cold", "warm"):
|
||||
t0 = time.time()
|
||||
audio = m.generate_audio(vs, text)
|
||||
dt = time.time() - t0
|
||||
a = audio.numpy() if hasattr(audio, "numpy") else np.asarray(audio)
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
secs = len(a) / m.sample_rate
|
||||
sf.write(os.path.join(OUT, f"{name}.wav"), a, m.sample_rate)
|
||||
print(f"[{name:6} {run:4}] gen={dt:5.2f}s audio={secs:5.2f}s RTF={dt/max(secs,0.01):.2f}", flush=True)
|
||||
|
||||
print("=== Whisper ===", flush=True)
|
||||
from faster_whisper import WhisperModel
|
||||
wm = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
for p in sorted(glob.glob(os.path.join(OUT, "*.wav"))):
|
||||
segs, _ = wm.transcribe(p, language="de", beam_size=5)
|
||||
print(os.path.basename(p), "::", " ".join(s.text.strip() for s in segs), flush=True)
|
||||
print("PRESET_DONE", flush=True)
|
||||
@@ -1,45 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, time, numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = os.path.join(BASE, "out_ptts"); os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
# Referenz ~18s wav fuer Klon
|
||||
y, _ = librosa.load(os.path.join(BASE, "ref.mp3"), sr=24000, mono=True)
|
||||
yt, _ = librosa.effects.trim(y, top_db=30)
|
||||
ref = os.path.join(BASE, "ptts_ref.wav")
|
||||
sf.write(ref, yt[:int(18*24000)], 24000)
|
||||
print(f"[ref] {min(len(yt)/24000,18):.1f}s", flush=True)
|
||||
|
||||
t0 = time.time()
|
||||
m = TTSModel.load_model(language="german_24l")
|
||||
print(f"[load] {time.time()-t0:.1f}s sr={m.sample_rate}", flush=True)
|
||||
t0 = time.time()
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
print(f"[clone] {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
for name, text in SENT.items():
|
||||
for run in ("cold", "warm"):
|
||||
t0 = time.time()
|
||||
audio = m.generate_audio(vs, text)
|
||||
dt = time.time() - t0
|
||||
a = audio.numpy() if hasattr(audio, "numpy") else np.asarray(audio, dtype=np.float32)
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
secs = len(a) / m.sample_rate
|
||||
p = os.path.join(OUT, f"{name}.wav"); sf.write(p, a, m.sample_rate)
|
||||
print(f"[{name:6} {run:4}] gen={dt:5.2f}s audio={secs:5.2f}s RTF={dt/max(secs,0.01):.2f}", flush=True)
|
||||
|
||||
print("=== Whisper ===", flush=True)
|
||||
from faster_whisper import WhisperModel
|
||||
wm = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
import glob
|
||||
for p in sorted(glob.glob(os.path.join(OUT, "*.wav"))):
|
||||
segs, _ = wm.transcribe(p, language="de", beam_size=5)
|
||||
print(os.path.basename(p), "::", " ".join(s.text.strip() for s in segs), flush=True)
|
||||
print("PTTS_DONE", flush=True)
|
||||
@@ -1,28 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Rendert benannte Test-WAVs auf den Desktop (zum Anhören). LABEL steuert den Datei-Präfix.
|
||||
Server muss auf :8134 laufen. Nutzung: LUCY_LABEL=nachher python render_desktop.py"""
|
||||
import os, json, urllib.request
|
||||
|
||||
SERVER = "http://127.0.0.1:8134/tts"
|
||||
LABEL = os.environ.get("LUCY_LABEL", "nachher")
|
||||
OUT = os.path.join(os.path.expanduser("~"), "Desktop", "lucy_test"); os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
TESTS = [
|
||||
("01_kurz", "Schön, dass du da bist, Commander."),
|
||||
("02_langer_satz","Also, Commander, wenn ich das kurz zusammenfassen darf: heute früh war die Verbindung weg, dann lief das Update durch, danach musste ich zweimal neu starten, und jetzt sind endlich alle Dienste oben, das Gedächtnis ist warm, und wir können ganz entspannt weitermachen."),
|
||||
("03_mittel", "Guten Morgen, Commander. Das nächtliche Backup ist um 3 Uhr durchgelaufen, es gab keine Fehler, und die Datenbank läuft mit 92 Prozent Auslastung völlig stabil."),
|
||||
("04_akronyme", "Die CPU der RX 9070 XT und die GPU laufen mit ROCm, sagt der PC über das LAN. Es ist 21 Uhr UTC."),
|
||||
("05_locker", "Klar, mach ich. Gib mir kurz zwei Sekunden, dann schau ich in den Logs nach, was da schiefgelaufen ist."),
|
||||
]
|
||||
|
||||
def render(text):
|
||||
req = urllib.request.Request(SERVER, data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"})
|
||||
return urllib.request.urlopen(req, timeout=180).read()
|
||||
|
||||
for name, text in TESTS:
|
||||
wav = render(text)
|
||||
path = os.path.join(OUT, f"{name}_{LABEL}.wav")
|
||||
open(path, "wb").write(wav)
|
||||
print(f"OK {os.path.basename(path)} ({len(wav)//1024} KB)")
|
||||
print(f"\nFertig -> {OUT}")
|
||||
@@ -1,20 +0,0 @@
|
||||
import time
|
||||
from llama_cpp import Llama
|
||||
|
||||
MODEL = r"F:\Coding Stuff\mission-control-2\client\lucy-tts\models\OuteTTS-1.0-1B-Q8_0.gguf"
|
||||
|
||||
t0 = time.time()
|
||||
llm = Llama(model_path=MODEL, n_gpu_layers=999, n_ctx=8192, verbose=True)
|
||||
print(f"[load] {time.time()-t0:.1f}s", flush=True)
|
||||
|
||||
prompt = "Guten Tag, mein Name ist"
|
||||
# warmup
|
||||
llm(prompt, max_tokens=16, temperature=0.8)
|
||||
# messen
|
||||
for i in range(2):
|
||||
t0 = time.time()
|
||||
out = llm(prompt, max_tokens=300, temperature=0.8)
|
||||
dt = time.time() - t0
|
||||
n = out["usage"]["completion_tokens"]
|
||||
print(f"[run {i+1}] {n} tokens in {dt:.2f}s = {n/dt:.1f} tok/s", flush=True)
|
||||
print("SPEED_DONE", flush=True)
|
||||
@@ -1,47 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Deutscher TTS-Stresstest: rendert 10 Sätze (typische DE-Fallen, alle Längen), transkribiert per
|
||||
Whisper (SOLL vs IST) und misst Pausen/Prosodie. Server muss auf :8134 laufen (pocket_server /tts)."""
|
||||
import os, io, json, urllib.request
|
||||
import numpy as np, soundfile as sf, librosa
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
SERVER = "http://127.0.0.1:8134/tts"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_de_stress"; os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
TESTS = [
|
||||
("01_kurz_umlaut", "kurz", "Schön, dass du da bist, Commander."),
|
||||
("02_kurz_zahlen", "kurz", "Es ist 21 Uhr 40, und wir haben noch 16 GB frei."),
|
||||
("03_komposita", "mittel", "Die Geschwindigkeitsbegrenzung und die Aufmerksamkeitsspanne sind heute besonders wichtig."),
|
||||
("04_ss_sch_ch", "mittel", "Ich weiß nicht, ob die Straße noch frei ist, aber ich schaue gleich mal nach."),
|
||||
("05_umlaut_haeufung", "mittel", "Über fünf Türme, größere Bäume und schöne Flüsse führt die Röhre nach Süden."),
|
||||
("06_fremdwoerter", "mittel", "Das Backup-Skript läuft im Terminal, der Router ist online und der Server antwortet sofort."),
|
||||
("07_lang_gemischt", "lang", "Guten Morgen, Commander. Das nächtliche Backup ist um 3 Uhr durchgelaufen, es gab keine Fehler, und die Datenbank läuft mit 92 Prozent Auslastung völlig stabil."),
|
||||
("08_sehr_lang", "sehr lang", "Also, Commander, wenn ich das kurz zusammenfassen darf: heute früh war die Verbindung weg, dann lief das Update durch, danach musste ich zweimal neu starten, und jetzt sind endlich alle Dienste oben, das Gedächtnis ist warm, und wir können ganz entspannt weitermachen."),
|
||||
("09_akronyme", "mittel", "Die CPU der RX 9070 XT und die GPU laufen mit ROCm, sagt der PC über das LAN."),
|
||||
("10_zungenbrecher", "kurz", "Der Frühstücksfleischklößchenlieferant frühstückt fröhlich am frühen Freitag."),
|
||||
]
|
||||
|
||||
def render(text):
|
||||
req = urllib.request.Request(SERVER, data=json.dumps({"text": text}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"})
|
||||
return urllib.request.urlopen(req, timeout=180).read()
|
||||
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
for name, length, text in TESTS:
|
||||
wav = render(text)
|
||||
path = os.path.join(OUT, name + ".wav")
|
||||
open(path, "wb").write(wav)
|
||||
a, sr = sf.read(io.BytesIO(wav)); a = np.asarray(a, dtype=np.float32).reshape(-1); dur = len(a) / sr
|
||||
segs, _ = w.transcribe(path, language="de", beam_size=5, word_timestamps=True)
|
||||
words = [x for s in segs for x in (s.words or [])]
|
||||
got = " ".join(x.word.strip() for x in words)
|
||||
pauses = [(words[i].word.strip(), words[i + 1].start - words[i].end)
|
||||
for i in range(len(words) - 1) if (words[i + 1].start - words[i].end) >= 0.25]
|
||||
f0 = librosa.yin(a, fmin=80, fmax=400, sr=sr, frame_length=1024); f0v = f0[(f0 > 90) & (f0 < 380)]
|
||||
med = np.median(f0v) if f0v.size else 0; rng = (np.percentile(f0v, 90) - np.percentile(f0v, 10)) if f0v.size else 0
|
||||
print(f"\n### {name} [{length}] {dur:.1f}s Tempo {len(words)/max(dur,.01):.1f}W/s F0 {med:.0f}Hz/Spanne {rng:.0f}")
|
||||
print(f"SOLL: {text}")
|
||||
print(f"IST : {got}")
|
||||
if pauses:
|
||||
print("PAUSEN: " + ", ".join(f'„{ww}"={gg*1000:.0f}ms' for ww, gg in sorted(pauses, key=lambda t: -t[1])[:6]))
|
||||
print("\n=== STRESSTEST FERTIG ===")
|
||||
@@ -1,105 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""sweep_b2.py — findet den B2-Sweet-Spot auf DIESER Maschine (Ryzen 9700X, 8C/16T).
|
||||
Variiert WORKERS × THREADS-pro-Worker und misst je Konfig auf einer langen, mehrsätzigen Antwort:
|
||||
- wall : Gesamt-Wall-Clock (Generierung der ganzen Antwort)
|
||||
- ttfb : Time-to-first-audio (Reaktionszeit = erster Satz fertig)
|
||||
- audio : erzeugte Audiolänge (variiert leicht, da temp>0 stochastisch)
|
||||
- RTF : wall/audio (längen-normalisiert -> fairer Durchsatz-Vergleich; <1 = schneller als Echtzeit)
|
||||
Nutzt die ECHTEN pocket_server-Funktionen (_worker_init/_warmup/_gen_sentences_ordered).
|
||||
Schreibt sweep_b2_result.json + eine Tabelle und nennt am Ende den Sweet Spot.
|
||||
"""
|
||||
import os, time, json, statistics as st
|
||||
import numpy as np, librosa
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pocket_tts import TTSModel
|
||||
import pocket_server as ps
|
||||
|
||||
BASE = ps.BASE
|
||||
LONG = ("Guten Morgen, Commander. "
|
||||
"Das nächtliche Backup ist sauber durchgelaufen und es gab keine Fehler. "
|
||||
"Der Dienst läuft stabil und die Engine antwortet zügig. "
|
||||
"Ich habe die Modelle vorgewärmt und die Latenz im Blick behalten. "
|
||||
"Die Protokolle zeigen keine Auffälligkeiten in den letzten Stunden. "
|
||||
"Wenn du möchtest, fasse ich die offenen Punkte für heute zusammen. "
|
||||
"Danach kümmere ich mich um die anstehenden Updates und melde mich wieder.")
|
||||
|
||||
# (workers, threads_pro_worker). Ziel: workers*threads ~ 6–8 (8 physische Kerne).
|
||||
CONFIGS = [("seriell", 0, 0), ("pool", 2, 4), ("pool", 2, 3),
|
||||
("pool", 3, 2), ("pool", 3, 3), ("pool", 4, 2)]
|
||||
REPS = 3
|
||||
|
||||
|
||||
def _measure(sents):
|
||||
"""Eine Durchführung: (wall, ttfb, audio_s) über den aktuell in STATE gesetzten Pfad."""
|
||||
t0 = time.time(); first = None; total = 0
|
||||
for i, a in enumerate(ps._gen_sentences_ordered(sents)):
|
||||
if i == 0:
|
||||
first = time.time() - t0
|
||||
total += a.size
|
||||
return time.time() - t0, first, total / 24000.0
|
||||
|
||||
|
||||
def run_serial(reps):
|
||||
m = TTSModel.load_model(language=ps.LANG, lsd_decode_steps=ps.LSD, temp=ps.TEMP,
|
||||
noise_clamp=ps.NOISE_CLAMP, quantize=ps.QUANTIZE)
|
||||
ref = ps._prep_ref()
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(ps.VOICE_ST)
|
||||
except Exception:
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
ra, _ = librosa.load(ref, sr=m.sample_rate, mono=True)
|
||||
ps.STATE.clear()
|
||||
ps.STATE.update(m=m, vs=vs, sr=m.sample_rate, ref_fp=ps._fingerprint(ra, m.sample_rate))
|
||||
sents = ps._split_sentences(LONG)
|
||||
rows = [_measure(sents) for _ in range(reps)]
|
||||
ps.STATE.clear(); del m
|
||||
return rows, len(sents)
|
||||
|
||||
|
||||
def run_pool(workers, threads, reps):
|
||||
os.environ["LUCY_WORKER_THREADS"] = str(threads) # vom Worker beim Spawn gelesen
|
||||
pool = ProcessPoolExecutor(max_workers=workers, initializer=ps._worker_init)
|
||||
list(pool.map(ps._warmup, range(workers)))
|
||||
ref = ps._prep_ref(); ra, _ = librosa.load(ref, sr=24000, mono=True)
|
||||
ps.STATE.clear()
|
||||
ps.STATE.update(sr=24000, ref_fp=ps._fingerprint(ra, 24000), pool=pool)
|
||||
sents = ps._split_sentences(LONG)
|
||||
rows = [_measure(sents) for _ in range(reps)]
|
||||
pool.shutdown(wait=True); ps.STATE.clear()
|
||||
return rows, len(sents)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
results = []
|
||||
print(f"Text: {len(LONG)} Zeichen, REPS={REPS}\n")
|
||||
print(f"{'config':>14} | {'wall':>6} {'ttfb':>6} {'audio':>6} {'RTF':>5}")
|
||||
print("-" * 48)
|
||||
for kind, w, t in CONFIGS:
|
||||
try:
|
||||
rows, nsent = (run_serial(REPS) if kind == "seriell" else run_pool(w, t, REPS))
|
||||
except Exception as e:
|
||||
print(f"{kind} w{w} t{t}: FEHLER {e}")
|
||||
continue
|
||||
wall = st.median([r[0] for r in rows])
|
||||
ttfb = st.median([r[1] for r in rows])
|
||||
audio = st.median([r[2] for r in rows])
|
||||
rtf = wall / max(audio, 0.01)
|
||||
label = "seriell(1×all)" if kind == "seriell" else f"{w}w×{t}t"
|
||||
print(f"{label:>14} | {wall:6.2f} {ttfb:6.2f} {audio:6.2f} {rtf:5.2f}")
|
||||
results.append(dict(label=label, kind=kind, workers=(1 if kind == "seriell" else w),
|
||||
threads=t, wall=wall, ttfb=ttfb, audio=audio, rtf=rtf, sents=nsent))
|
||||
|
||||
json.dump(results, open(os.path.join(BASE, "sweep_b2_result.json"), "w"), indent=2)
|
||||
|
||||
pool_rows = [r for r in results if r["kind"] == "pool"]
|
||||
if pool_rows:
|
||||
best_tput = min(pool_rows, key=lambda r: r["rtf"])
|
||||
best_ttfb = min(pool_rows, key=lambda r: r["ttfb"])
|
||||
# Sweet Spot: bester Durchsatz, aber TTFB nicht >20% über dem TTFB-Sieger (Reaktion zählt)
|
||||
cand = [r for r in pool_rows if r["ttfb"] <= best_ttfb["ttfb"] * 1.20]
|
||||
sweet = min(cand, key=lambda r: r["rtf"]) if cand else best_tput
|
||||
print("\n>> bester Durchsatz :", best_tput["label"], f"(RTF {best_tput['rtf']:.2f})")
|
||||
print(">> beste Reaktion :", best_ttfb["label"], f"(TTFB {best_ttfb['ttfb']:.2f}s)")
|
||||
print(">> SWEET SPOT :", sweet["label"],
|
||||
f"-> LUCY_WORKERS={sweet['workers']} LUCY_WORKER_THREADS={sweet['threads']}")
|
||||
print("SWEEP_DONE")
|
||||
@@ -1,53 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Qualitäts-Sweep: noise_clamp (nie getestet, Hypothese: dämpft Artefakte+Kollaps) × lsd, temp 0.9.
|
||||
Produktions-äquivalente Verarbeitung (LEAD + cleanup v4 wie der Server) -> faires A/B gegen lucy_final."""
|
||||
import os, time, numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_quality_sweep"; os.makedirs(OUT, exist_ok=True)
|
||||
src, _ = librosa.load(os.path.join(BASE, "ref.mp3"), sr=24000, mono=True)
|
||||
srt, _ = librosa.effects.trim(src, top_db=30); ref = os.path.join(BASE, "ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
LEAD, TARGET_RMS = "Tja. ", 0.09
|
||||
SENT = {"kurz":"Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang":"Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft."}
|
||||
|
||||
def crop_lead(a, sr):
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) >= 2:
|
||||
cut = max(iv[0][1], iv[1][0] - int(0.06*sr)); a = a[cut:]
|
||||
return a
|
||||
def cleanup(a, sr):
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if a.size == 0: return a
|
||||
a = crop_lead(a, sr)
|
||||
rev,_ = librosa.effects.trim(a[::-1], top_db=45); a = rev[::-1] if rev.size else a
|
||||
rms = float(np.sqrt(np.mean(a**2))) or 1e-9; a = a*(TARGET_RMS/rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > 0.9: a = a*(0.9/peak)
|
||||
fi = min(int(0.008*sr), a.size//2)
|
||||
if fi>0: a[:fi]*=np.linspace(0.,1.,fi,dtype=np.float32); a[-fi:]*=np.linspace(1.,0.,fi,dtype=np.float32)
|
||||
pad = np.zeros(int(0.08*sr), dtype=np.float32)
|
||||
return np.concatenate([pad, a, pad])
|
||||
|
||||
# (tag, lsd, noise_clamp)
|
||||
COMBOS = [
|
||||
("A_baseline_lsd6_ncNone", 6, None),
|
||||
("B_lsd6_nc2", 6, 2.0),
|
||||
("C_lsd6_nc3", 6, 3.0),
|
||||
("D_lsd10_nc3", 10, 3.0),
|
||||
]
|
||||
for tag, lsd, nc in COMBOS:
|
||||
t0 = time.time()
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=lsd, temp=0.9, noise_clamp=nc)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr = m.sample_rate
|
||||
print(f"== {tag} (load {time.time()-t0:.1f}s) ==", flush=True)
|
||||
for name, text in SENT.items():
|
||||
t0 = time.time(); a = m.generate_audio(vs, LEAD+text, frames_after_eos=4); dt = time.time()-t0
|
||||
a = a.numpy() if hasattr(a,"numpy") else np.asarray(a)
|
||||
a = cleanup(a, sr); secs = a.size/sr
|
||||
sf.write(os.path.join(OUT, f"{tag}_{name}.wav"), a, sr)
|
||||
print(f" [{name:6}] gen={dt:4.1f}s audio={secs:4.1f}s RTF={dt/max(secs,0.01):.2f}", flush=True)
|
||||
print("SWEEP_DONE", flush=True)
|
||||
@@ -1,45 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Sweep 2 (Feinschliff): lsd 10 (glättet Rauschen) × noise_clamp 2/2.5/3, temp 0.9.
|
||||
Ziel: sauberes 'Commander' (nicht erzwungen, = niedrigeres nc) + wenig Rauschen (= höheres lsd)."""
|
||||
import os, time, numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_sweep2"; os.makedirs(OUT, exist_ok=True)
|
||||
src, _ = librosa.load(os.path.join(BASE, "ref.mp3"), sr=24000, mono=True)
|
||||
srt, _ = librosa.effects.trim(src, top_db=30); ref = os.path.join(BASE, "ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
LEAD, TARGET_RMS = "Tja. ", 0.09
|
||||
SENT = {"kurz":"Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang":"Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft."}
|
||||
|
||||
def crop_lead(a, sr):
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) >= 2: a = a[max(iv[0][1], iv[1][0]-int(0.06*sr)):]
|
||||
return a
|
||||
def cleanup(a, sr):
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if a.size == 0: return a
|
||||
a = crop_lead(a, sr)
|
||||
rev,_ = librosa.effects.trim(a[::-1], top_db=45); a = rev[::-1] if rev.size else a
|
||||
rms = float(np.sqrt(np.mean(a**2))) or 1e-9; a = a*(TARGET_RMS/rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > 0.9: a = a*(0.9/peak)
|
||||
fi = min(int(0.008*sr), a.size//2)
|
||||
if fi>0: a[:fi]*=np.linspace(0.,1.,fi,dtype=np.float32); a[-fi:]*=np.linspace(1.,0.,fi,dtype=np.float32)
|
||||
return np.concatenate([np.zeros(int(0.08*sr),dtype=np.float32), a, np.zeros(int(0.08*sr),dtype=np.float32)])
|
||||
|
||||
for nc in [2.0, 2.5, 3.0]:
|
||||
tag = f"lsd10_nc{str(nc).replace('.','p')}"
|
||||
t0 = time.time()
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=10, temp=0.9, noise_clamp=nc)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr = m.sample_rate
|
||||
print(f"== {tag} (load {time.time()-t0:.1f}s) ==", flush=True)
|
||||
for name, text in SENT.items():
|
||||
t0 = time.time(); a = m.generate_audio(vs, LEAD+text, frames_after_eos=4); dt = time.time()-t0
|
||||
a = a.numpy() if hasattr(a,"numpy") else np.asarray(a)
|
||||
a = cleanup(a, sr); secs = a.size/sr
|
||||
sf.write(os.path.join(OUT, f"{tag}_{name}.wav"), a, sr)
|
||||
print(f" [{name:6}] gen={dt:4.1f}s audio={secs:4.1f}s RTF={dt/max(secs,0.01):.2f}", flush=True)
|
||||
print("SWEEP2_DONE", flush=True)
|
||||
@@ -1,28 +0,0 @@
|
||||
"""Prüft normalize_numbers: Einheiten/Uhrzeiten/Dezimalzahlen verbalisieren, Modellnummern schützen."""
|
||||
from text_norm import normalize_numbers as N
|
||||
|
||||
def check(name, text, must_have=(), must_not=()):
|
||||
out = N(text)
|
||||
ok = all(h in out for h in must_have) and all(x not in out for x in must_not)
|
||||
print(f"{'PASS' if ok else 'FAIL'} {name}: \"{text}\" -> \"{out}\"")
|
||||
return ok
|
||||
|
||||
allok = True
|
||||
allok &= check("Uhrzeit", "Es ist 18:01 Uhr.", must_have=["achtzehn Uhr", "eins"], must_not=[":", "18:"])
|
||||
allok &= check("Uhrzeit :00", "Um 18:00.", must_have=["achtzehn Uhr"], must_not=[":"])
|
||||
allok &= check("Uhrzeit HH:MM", "Treffen um 9:30 Uhr.", must_have=["neun Uhr", "dreißig"], must_not=[":"])
|
||||
allok &= check("Einheit GB verbalisiert", "Sie hat 16 GB Speicher.", must_have=["sechzehn GB"], must_not=[" 16 "])
|
||||
allok &= check("Einheit TB + GB", "8 TB und 32 GB RAM.", must_have=["acht TB", "zweiunddreißig GB", "RAM"])
|
||||
allok &= check("Dezimal verbalisiert", "Läuft mit 4.5 GHz.", must_have=["vier Komma", "fünf", "GHz"], must_not=["4.5"])
|
||||
allok &= check("Modellnummer geschützt", "Die RX 9070 XT ist schnell.", must_have=["9070", "XT"], must_not=["neuntausend"])
|
||||
allok &= check("Version geschützt", "Qwen3.6 nutzt Version 2.", must_have=["Qwen3.6", "zwei"], must_not=["Qwen3.sechs"])
|
||||
allok &= check("Modell H100 geschützt", "Das braucht eine H100.", must_have=["H100"])
|
||||
allok &= check("Zahl im Text", "Ich habe 5 Äpfel.", must_have=["fünf", "Äpfel"], must_not=[" 5 "])
|
||||
allok &= check("Prozent", "Das sind 50 Prozent.", must_have=["fünfzig Prozent"])
|
||||
allok &= check("Grad", "Es sind 25 Grad.", must_have=["fünfundzwanzig Grad"])
|
||||
allok &= check("Jahr", "Im Jahr 2026.", must_have=["zweitausend"], must_not=["2026"])
|
||||
allok &= check("Datum-Ordinal", "Am 3. Januar.", must_have=["Januar"], must_not=["3. Januar", " 3 "])
|
||||
allok &= check("große ID roh", "Fehler 123456.", must_have=["123456"])
|
||||
|
||||
print("\nALLE TESTS GRÜN" if allok else "\nES GAB FEHLER")
|
||||
raise SystemExit(0 if allok else 1)
|
||||
@@ -1,120 +0,0 @@
|
||||
"""Deutsche Zahlen-/Uhrzeit-Normalisierung für Lucys Stimme (num2words).
|
||||
|
||||
Eigenständig (nur re + num2words, KEIN torch/pocket_tts-Import), damit schnell testbar und von
|
||||
pocket_server importierbar.
|
||||
|
||||
Kernidee: Zahlen im Fließtext verbalisieren ("18:01" -> "achtzehn Uhr eins", "16 GB" -> "sechzehn
|
||||
GB", "4.5" -> "vier Komma fünf"), aber Modell-/Versionsnummern SCHÜTZEN ("RX 9070 XT", "Qwen3.6",
|
||||
"H100" bleiben roh — die soll man nicht als Kardinalzahl lesen). Abschaltbar via LUCY_NUM_NORM=0.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
|
||||
try:
|
||||
from num2words import num2words as _n2w
|
||||
except Exception: # Lib fehlt -> No-op (Stimme läuft trotzdem)
|
||||
_n2w = None
|
||||
|
||||
# AUS per Default (2026-07-01, empirisch via Whisper-Rücktranskription verifiziert): Pocket german_24l
|
||||
# spricht ROHE Ziffern ("21 Uhr 40", "16 GB", "4.5 GHz", "9:30") sauber. Vorverbalisierte deutsche
|
||||
# Kompositzahlen ("einundzwanzig") ZERBRICHT das Modell zu Kauderwelsch ("ein Mund", "ein Aus-20").
|
||||
# Also NICHT verbalisieren. Mit LUCY_NUM_NORM=1 wieder anschaltbar (falls je ein Modell es braucht).
|
||||
NUM_NORM = os.environ.get("LUCY_NUM_NORM", "0") not in ("0", "false", "False")
|
||||
|
||||
_MONTHS = "Januar Februar März April Mai Juni Juli August September Oktober November Dezember".split()
|
||||
_MONTH_RX = "|".join(_MONTHS)
|
||||
# FULLY-uppercase 2+-Einheiten, neben denen Zahlen VERBALISIERT werden (sechzehn GB) — im Gegensatz
|
||||
# zu Modell-Akronymen (RX/XT/RTX), wo die Ziffern roh bleiben (RX 9070 XT). Gemischt-Case-Einheiten
|
||||
# (GHz/MHz/kg/km/Grad/Prozent) werden von der Akronym-Regel eh nicht erfasst -> dort wird ohnehin verbalisiert.
|
||||
_CAPS_UNITS = {"GB", "TB", "MB", "KB", "PB", "EB", "KW", "MW"}
|
||||
|
||||
|
||||
# Garble-Kandidaten (Whisper-verifiziert): Pocket german_24l kann diese Akronyme/Codes NICHT sauber
|
||||
# sprechen ("ROCm"->"AOC HM", "UTC"->Kauderwelsch) -> Ersatz durch gesprochene/buchstabierte Form.
|
||||
# Erweiterbar via LUCY_ACRONYM_EXTRA="ABC=A B C,XYZ=..." (neue Fälle findet transcribe.py).
|
||||
_ACRONYM_FIX = {
|
||||
"ROCm": "Rockem", "UTC": "U T C", "GMT": "G M T",
|
||||
"CEST": "C E S T", "CET": "C E T", "MESZ": "M E S Z", "MEZ": "M E Z",
|
||||
"NVMe": "N V M E", "PCIe": "P C I E",
|
||||
}
|
||||
|
||||
def _build_acr():
|
||||
m = dict(_ACRONYM_FIX)
|
||||
for pair in os.environ.get("LUCY_ACRONYM_EXTRA", "").split(","):
|
||||
if "=" in pair:
|
||||
k, v = pair.split("=", 1)
|
||||
if k.strip():
|
||||
m[k.strip()] = v.strip()
|
||||
return m
|
||||
|
||||
_ACR_MAP = _build_acr()
|
||||
_ACR_RX = re.compile(r"\b(" + "|".join(re.escape(k) for k in sorted(_ACR_MAP, key=len, reverse=True)) + r")\b") if _ACR_MAP else None
|
||||
|
||||
ACR_ON = os.environ.get("LUCY_ACRONYM", "1") not in ("0", "false", "False") # A/B-Abschaltung
|
||||
|
||||
def fix_acronyms(text: str) -> str:
|
||||
"""Garble-anfällige Akronyme durch gesprochene Form ersetzen (ROCm -> Rockem, UTC -> U T C)."""
|
||||
return _ACR_RX.sub(lambda m: _ACR_MAP[m.group(0)], text) if (_ACR_RX and ACR_ON) else text
|
||||
|
||||
|
||||
def _card(v) -> str:
|
||||
try:
|
||||
return _n2w(int(v), lang="de")
|
||||
except Exception:
|
||||
return str(v)
|
||||
|
||||
|
||||
def normalize_numbers(text: str) -> str:
|
||||
if not NUM_NORM or _n2w is None:
|
||||
return text
|
||||
|
||||
masks: list[str] = []
|
||||
def _mask(s: str) -> str:
|
||||
masks.append(s)
|
||||
return f"\x00{len(masks) - 1}\x00"
|
||||
|
||||
# 1) Uhrzeiten: "18:01 Uhr"/"18:01" -> "18 Uhr 01", "18:00" -> "18 Uhr" (Ziffern verbalisiert Schritt 6)
|
||||
text = re.sub(r"\b(\d{1,2}):00(\s*Uhr)?\b", r"\1 Uhr", text)
|
||||
text = re.sub(r"\b(\d{1,2}):(\d{2})\s*Uhr\b", r"\1 Uhr \2", text)
|
||||
text = re.sub(r"\b(\d{1,2}):(\d{2})\b", r"\1 Uhr \2", text)
|
||||
|
||||
# 2) Datums-Ordinalzahlen: "3. Januar" -> "dritter Januar"
|
||||
def _ord(m):
|
||||
try:
|
||||
return f"{_n2w(int(m.group(1)), to='ordinal', lang='de')} {m.group(2)}"
|
||||
except Exception:
|
||||
return m.group(0)
|
||||
text = re.sub(rf"\b(\d{{1,2}})\.\s+({_MONTH_RX})\b", _ord, text)
|
||||
|
||||
# 3) Modell-/Versions-Token (Buchstabe UND Ziffer, evtl. mit Punkten: Qwen3.6, H100, v2.1, RTX4090)
|
||||
# -> ganzes Token maskieren (nie verbalisieren). Satz-Endpunkt/-Komma bleibt draußen.
|
||||
def _mask_model(m):
|
||||
tok = m.group(0); trail = ""
|
||||
while tok and tok[-1] in ".,":
|
||||
trail = tok[-1] + trail; tok = tok[:-1]
|
||||
if re.search(r"[A-Za-zÄÖÜäöü]", tok) and re.search(r"\d", tok):
|
||||
return _mask(tok) + trail
|
||||
return m.group(0)
|
||||
text = re.sub(r"[A-Za-zÄÖÜäöü0-9]+(?:[.,][A-Za-zÄÖÜäöü0-9]+)*", _mask_model, text)
|
||||
|
||||
# 4) Standalone-Dezimalzahl (nur Ziffern, kein Buchstabe): "4.5" -> "vier Komma fünf"
|
||||
# (Nachkommastellen ziffernweise, wie im Deutschen üblich).
|
||||
def _dec(m):
|
||||
return f"{_card(m.group(1))} Komma {' '.join(_card(d) for d in m.group(2))}"
|
||||
text = re.sub(r"(?<![\w\x00])(\d+)[.,](\d+)(?![\w\x00])", _dec, text)
|
||||
|
||||
# 5) Zahl neben ALL-CAPS-Token: Einheit (GB/TB..) -> verbalisieren; Modell-Akronym (RX/XT) -> roh maskieren.
|
||||
def _by_ctx(num, acr):
|
||||
return num if acr.upper() in _CAPS_UNITS else _mask(num)
|
||||
text = re.sub(r"\b([A-ZÄÖÜ]{2,})\s+(\d{1,6})\b", lambda m: f"{m.group(1)} {_by_ctx(m.group(2), m.group(1))}", text)
|
||||
text = re.sub(r"\b(\d{1,6})\s+([A-ZÄÖÜ]{2,})\b", lambda m: f"{_by_ctx(m.group(1), m.group(2))} {m.group(2)}", text)
|
||||
|
||||
# 6) verbleibende reine Ganzzahlen (0..9999) verbalisieren; größere roh lassen (IDs/Codes)
|
||||
def _c(m):
|
||||
v = int(m.group(0))
|
||||
return _n2w(v, lang="de") if v <= 9999 else m.group(0)
|
||||
text = re.sub(r"(?<![\w\x00])\d{1,4}(?![\w\x00])", _c, text)
|
||||
|
||||
# 7) Masken zurückholen
|
||||
text = re.sub(r"\x00(\d+)\x00", lambda m: masks[int(m.group(1))], text)
|
||||
return text
|
||||
@@ -1,19 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Whisper-Rücktranskription: WAS sagt die Stimme WIRKLICH? Liest die generierten Wörter, damit man
|
||||
beim Fine-Tunen SEHEN kann, wo Pocket Zahlen/Wörter vermurkst (statt nur zu hören/raten).
|
||||
|
||||
Nutzung: python transcribe.py [ordner-mit-wavs]
|
||||
Default-Ordner: C:\\Users\\TobisPC\\Desktop\\lucy_pocket_tune
|
||||
"""
|
||||
import os, sys, glob
|
||||
import soundfile as sf
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
DIR = sys.argv[1] if len(sys.argv) > 1 else r"C:\Users\TobisPC\Desktop\lucy_pocket_tune"
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
for p in sorted(glob.glob(os.path.join(DIR, "*.wav"))):
|
||||
a, sr = sf.read(p)
|
||||
seg, _ = w.transcribe(p, language="de", beam_size=5)
|
||||
txt = " ".join(s.text for s in seg).strip()
|
||||
print(f"[{os.path.basename(p)}] ({len(a)/sr:.1f}s)")
|
||||
print(f" HÖRT WHISPER: {txt}\n")
|
||||
@@ -1,50 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""ttfb_test.py — misst die Stream-TTFB im SERIELLEN Pfad mit kurzem erstem Chunk an/aus.
|
||||
Ein Modell-Load, dann je Modus 3 Durchläufe -> Median. Belegt, ob keep_first_short Lucys
|
||||
Sprechbeginn beschleunigt, ohne den Rest zu verschlechtern.
|
||||
"""
|
||||
import time, statistics as st
|
||||
import numpy as np, librosa
|
||||
from pocket_tts import TTSModel
|
||||
import pocket_server as ps
|
||||
|
||||
LONG = ("Guten Morgen, Commander. "
|
||||
"Das nächtliche Backup ist sauber durchgelaufen und es gab keine Fehler. "
|
||||
"Der Dienst läuft stabil und die Engine antwortet zügig. "
|
||||
"Ich habe die Modelle vorgewärmt und die Latenz im Blick behalten. "
|
||||
"Die Protokolle zeigen keine Auffälligkeiten in den letzten Stunden. "
|
||||
"Wenn du möchtest, fasse ich die offenen Punkte für heute zusammen. "
|
||||
"Danach kümmere ich mich um die anstehenden Updates und melde mich wieder.")
|
||||
|
||||
m = TTSModel.load_model(language=ps.LANG, lsd_decode_steps=ps.LSD, temp=ps.TEMP,
|
||||
noise_clamp=ps.NOISE_CLAMP, quantize=ps.QUANTIZE)
|
||||
ref = ps._prep_ref()
|
||||
try:
|
||||
vs = m.get_state_for_audio_prompt(ps.VOICE_ST)
|
||||
except Exception:
|
||||
vs = m.get_state_for_audio_prompt(ref)
|
||||
ra, _ = librosa.load(ref, sr=m.sample_rate, mono=True)
|
||||
ps.STATE.clear()
|
||||
ps.STATE.update(m=m, vs=vs, sr=m.sample_rate, ref_fp=ps._fingerprint(ra, m.sample_rate))
|
||||
|
||||
|
||||
def measure(keep_first):
|
||||
sents = ps._split_sentences(LONG, keep_first_short=keep_first)
|
||||
t0 = time.time(); first = None; total = 0
|
||||
for i, a in enumerate(ps._gen_sentences_ordered(sents)):
|
||||
if i == 0:
|
||||
first = time.time() - t0
|
||||
total += a.size
|
||||
return first, time.time() - t0, total / 24000.0, len(sents), sents[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for keep in (False, True):
|
||||
ttfbs, walls = [], []
|
||||
n = s0 = None
|
||||
for _ in range(3):
|
||||
f, w, au, n, s0 = measure(keep)
|
||||
ttfbs.append(f); walls.append(w)
|
||||
print(f"keep_first_short={str(keep):5} | TTFB={st.median(ttfbs):4.2f}s wall={st.median(walls):5.2f}s "
|
||||
f"chunks={n} erster='{s0[:38]}'")
|
||||
print("TTFB_DONE")
|
||||
@@ -1,79 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Fix: (1) 'vorne abgeschnitten' = Modell startet mid-Phonem -> Lead-Wort voranstellen,
|
||||
in flow generieren, dann in der STILLE-LÜCKE davor schneiden (voller Onset bleibt).
|
||||
(2) 'zu laut' = RMS-Normalisierung auf Zielpegel statt Peak 0.95."""
|
||||
import os, numpy as np, soundfile as sf, librosa
|
||||
from pocket_tts import TTSModel
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-tts"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_front_loud"; os.makedirs(OUT, exist_ok=True)
|
||||
src, _ = librosa.load(os.path.join(BASE, "ref.mp3"), sr=24000, mono=True)
|
||||
srt, _ = librosa.effects.trim(src, top_db=30); ref = os.path.join(BASE, "ref.wav")
|
||||
sf.write(ref, srt[:int(18*24000)], 24000)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
LEAD = "Tja. " # Wegwerf-Lead -> erzeugt natürlichen Onset fürs echte erste Wort
|
||||
|
||||
def drop_tail_blip(a, sr):
|
||||
for _ in range(3):
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) < 2: break
|
||||
srms = float(np.median([np.sqrt(np.mean(a[s:e]**2)) for s,e in iv[:-1]]))
|
||||
s,e = iv[-1]; dur=(e-s)/sr; rms=float(np.sqrt(np.mean(a[s:e]**2))); gap=(s-iv[-2][1])/sr
|
||||
if gap>0.35 and rms<0.30*srms and dur<0.35: a = a[:iv[-2][1]]
|
||||
else: break
|
||||
return a
|
||||
|
||||
def crop_lead(a, sr):
|
||||
"""Schneide in der Lücke NACH dem Lead-Wort -> echtes 1. Wort voll erhalten."""
|
||||
iv = librosa.effects.split(a, top_db=35)
|
||||
if len(iv) >= 2:
|
||||
# Ende des 1. (Lead-)Segments + kleiner Sicherheitsabstand in die Lücke
|
||||
cut = iv[0][1] + int(0.03*sr)
|
||||
nxt = iv[1][0]
|
||||
cut = min(cut, max(iv[0][1], nxt - int(0.04*sr))) # mind. 40ms Stille vor echtem Wort lassen
|
||||
a = a[cut:]
|
||||
return a
|
||||
|
||||
def finalize(a, sr, target_rms, peak_cap=0.9, front_trim=False):
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if front_trim:
|
||||
yt,_ = librosa.effects.trim(a, top_db=45); a = yt if yt.size else a
|
||||
else:
|
||||
# nur HINTEN trimmen (vorne unangetastet lassen)
|
||||
rev,_ = librosa.effects.trim(a[::-1], top_db=45); a = rev[::-1] if rev.size else a
|
||||
# RMS-Normalisierung auf Zielpegel
|
||||
rms = float(np.sqrt(np.mean(a**2))) or 1e-9
|
||||
a = a * (target_rms / rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > peak_cap: a = a * (peak_cap / peak) # Sicherheits-Clamp
|
||||
fi = min(int(0.008*sr), a.size//2)
|
||||
if fi>0:
|
||||
a[:fi]*=np.linspace(0.,1.,fi,dtype=np.float32); a[-fi:]*=np.linspace(1.,0.,fi,dtype=np.float32)
|
||||
pad = np.zeros(int(0.08*sr), dtype=np.float32)
|
||||
return np.concatenate([pad, a, pad])
|
||||
|
||||
m = TTSModel.load_model(language="german_24l", lsd_decode_steps=6, temp=0.9)
|
||||
vs = m.get_state_for_audio_prompt(ref); sr = m.sample_rate
|
||||
w = WhisperModel("small", device="cpu", compute_type="int8")
|
||||
|
||||
for name, text in SENT.items():
|
||||
# Onset-Fix-Roh: mit Lead generieren, Tail-Blip weg, Lead wegschneiden
|
||||
raw = m.generate_audio(vs, LEAD + text, frames_after_eos=4)
|
||||
raw = raw.numpy() if hasattr(raw,"numpy") else np.asarray(raw)
|
||||
raw = np.asarray(raw, dtype=np.float32).reshape(-1)
|
||||
raw = drop_tail_blip(raw, sr)
|
||||
cropped = crop_lead(raw, sr)
|
||||
for tr in [0.12, 0.09, 0.06]:
|
||||
out = finalize(cropped, sr, tr, front_trim=False)
|
||||
sf.write(os.path.join(OUT, f"{name}_onset_rms{int(tr*100):02d}.wav"), out, sr)
|
||||
# Whisper-Check: Output darf NICHT mit Lead beginnen
|
||||
sf.write(os.path.join(OUT, f"_chk_{name}.wav"), finalize(cropped, sr, 0.09), sr)
|
||||
seg,_ = w.transcribe(os.path.join(OUT, f"_chk_{name}.wav"), language="de", beam_size=5)
|
||||
txt = " ".join(s.text for s in seg).strip()
|
||||
print(f"[{name}] Whisper-Start: {txt[:55]!r}", flush=True)
|
||||
print("FRONT_LOUD_DONE", flush=True)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user