From e19831f7d5f3e0a860e713bee3b6aed50dee8069 Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Fri, 26 Jun 2026 22:02:52 +0200 Subject: [PATCH] Feat: Hermes Voice Client (Wake Word + STT + Vision + TTS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows-Desktop-Script: Energy VAD + Whisper Wake Detection, faster-whisper STT, mss Screen Capture, MC2 Vision/Chat API, Edge TTS Ausgabe, pystray System Tray. Kein Account nötig — Wake Word frei konfigurierbar via WAKE_WORDS in config.py. Co-Authored-By: Claude Sonnet 4.6 --- client/hermes-voice/ai_client.py | 45 ++++++++ client/hermes-voice/audio_input.py | 154 ++++++++++++++++++++++++++ client/hermes-voice/audio_output.py | 58 ++++++++++ client/hermes-voice/config.py | 62 +++++++++++ client/hermes-voice/main.py | 129 +++++++++++++++++++++ client/hermes-voice/requirements.txt | 28 +++++ client/hermes-voice/screen_capture.py | 19 ++++ client/hermes-voice/setup.bat | 40 +++++++ client/hermes-voice/start.bat | 4 + 9 files changed, 539 insertions(+) create mode 100644 client/hermes-voice/ai_client.py create mode 100644 client/hermes-voice/audio_input.py create mode 100644 client/hermes-voice/audio_output.py create mode 100644 client/hermes-voice/config.py create mode 100644 client/hermes-voice/main.py create mode 100644 client/hermes-voice/requirements.txt create mode 100644 client/hermes-voice/screen_capture.py create mode 100644 client/hermes-voice/setup.bat create mode 100644 client/hermes-voice/start.bat diff --git a/client/hermes-voice/ai_client.py b/client/hermes-voice/ai_client.py new file mode 100644 index 0000000..ddbe48f --- /dev/null +++ b/client/hermes-voice/ai_client.py @@ -0,0 +1,45 @@ +import httpx +from config import ( + MC2_BASE_URL, CHAT_MODEL, VISION_MODEL, + MAX_RESPONSE_TOKENS, REQUEST_TIMEOUT, SYSTEM_PROMPT, + SCREENSHOT_ON_QUERY, +) + + +def ask(text: str, screenshot_b64: str | None = None) -> str: + """Schickt Text (+ optionalen Screenshot) an MC2 und gibt die Antwort zurück.""" + use_vision = screenshot_b64 is not None and SCREENSHOT_ON_QUERY + model = VISION_MODEL if use_vision else CHAT_MODEL + + if use_vision: + user_content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": { + "url": f"data:image/jpeg;base64,{screenshot_b64}" + }}, + ] + else: + user_content = text + + payload = { + "model": model, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_content}, + ], + "max_tokens": MAX_RESPONSE_TOKENS, + "stream": False, + } + + try: + with httpx.Client(timeout=REQUEST_TIMEOUT) as client: + response = client.post( + f"{MC2_BASE_URL}/chat/completions", + json=payload, + ) + response.raise_for_status() + return response.json()["choices"][0]["message"]["content"].strip() + except httpx.TimeoutException: + return "Entschuldigung, die Antwort hat zu lange gedauert." + except Exception as e: + return f"Verbindungsfehler: {e}" diff --git a/client/hermes-voice/audio_input.py b/client/hermes-voice/audio_input.py new file mode 100644 index 0000000..21844ac --- /dev/null +++ b/client/hermes-voice/audio_input.py @@ -0,0 +1,154 @@ +""" +Audio-Input: Energy VAD → Whisper Wake Detection → Whisper STT + +Ablauf: + 1. Energie-VAD erkennt Sprache im Mikrofon + 2. Whisper tiny transkribiert den Clip (schnell) + 3. Enthält das Transcript ein Wake Word? → Ding + Befehl aufnehmen + 4. Enthält es Wake Word + Befehl in einem Atemzug? → direkt zurückgeben +""" + +import os +import threading +import tempfile +import numpy as np +import sounddevice as sd +import scipy.io.wavfile as wav +from faster_whisper import WhisperModel +from config import ( + WAKE_WORDS, WAKE_WHISPER_MODEL, + WHISPER_MODEL_SIZE, WHISPER_LANGUAGE, + SILENCE_TIMEOUT, MAX_RECORD_SECONDS, +) + +SAMPLE_RATE = 16000 +CHUNK = 1024 + +# Energie-Schwelle: unter diesem RMS = Stille +# Ggf. anpassen wenn zu sensitiv (höher) oder zu träge (niedriger) +ENERGY_THRESHOLD = 0.008 + +_whisper_tiny: WhisperModel | None = None +_whisper_main: WhisperModel | None = None + + +def _get_whisper(size: str) -> WhisperModel: + global _whisper_tiny, _whisper_main + if size == WAKE_WHISPER_MODEL: + if _whisper_tiny is None: + print(f"[STT] Lade Whisper {size} (Wake Detection)…") + _whisper_tiny = WhisperModel(size, device="cpu", compute_type="int8") + return _whisper_tiny + else: + if _whisper_main is None: + print(f"[STT] Lade Whisper {size} (Befehl-Transkription)…") + _whisper_main = WhisperModel(size, device="cpu", compute_type="int8") + return _whisper_main + + +def _record_until_silence( + stop_event: threading.Event, + max_seconds: float = 6.0, + silence_timeout: float = 1.5, +) -> np.ndarray | None: + """Nimmt Audio auf bis zur Stille oder Timeout. Gibt None zurück wenn gestoppt.""" + frames: list[np.ndarray] = [] + speech_chunks = 0 + silence_chunks = 0 + silence_limit = int(silence_timeout * SAMPLE_RATE / CHUNK) + max_chunks = int(max_seconds * SAMPLE_RATE / CHUNK) + + with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, + dtype="float32", blocksize=CHUNK) as stream: + for _ in range(max_chunks): + if stop_event.is_set(): + return None + data, _ = stream.read(CHUNK) + chunk = data[:, 0] + frames.append(chunk.copy()) + energy = float(np.sqrt(np.mean(chunk ** 2))) + + if energy > ENERGY_THRESHOLD: + speech_chunks += 1 + silence_chunks = 0 + else: + silence_chunks += 1 + + if speech_chunks > 2 and silence_chunks >= silence_limit: + break + + if speech_chunks < 2: + return None # nur Rauschen, kein echter Sprachinhalt + + return np.concatenate(frames) + + +def _transcribe(audio: np.ndarray, model_size: str) -> str: + """Schreibt Audio-Array als WAV, transkribiert mit Whisper.""" + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + tmp = f.name + try: + wav.write(tmp, SAMPLE_RATE, (audio * 32767).astype(np.int16)) + model = _get_whisper(model_size) + segments, _ = model.transcribe( + tmp, + language=WHISPER_LANGUAGE, + beam_size=3, + vad_filter=True, + ) + return " ".join(s.text for s in segments).strip() + finally: + os.unlink(tmp) + + +def _strip_wake_word(text: str) -> str: + """Entfernt das Wake Word vom Anfang des Textes.""" + lower = text.lower() + for ww in sorted(WAKE_WORDS, key=len, reverse=True): + idx = lower.find(ww) + if idx != -1: + rest = text[idx + len(ww):].lstrip(" ,.") + return rest + return text + + +def wait_for_wake_word(stop_event: threading.Event) -> bool: + """ + Lauscht kontinuierlich. Gibt True zurück wenn Wake Word erkannt, False wenn gestoppt. + Initialisiert Whisper-Modelle beim ersten Aufruf. + """ + _get_whisper(WAKE_WHISPER_MODEL) # Modell vorladen + print(f"[Wake] Höre auf: {WAKE_WORDS}") + + while not stop_event.is_set(): + audio = _record_until_silence(stop_event, max_seconds=6.0, silence_timeout=1.0) + if audio is None: + continue + + text = _transcribe(audio, WAKE_WHISPER_MODEL) + if not text: + continue + + lower = text.lower() + if any(ww in lower for ww in WAKE_WORDS): + return True + + return False + + +def record_speech() -> np.ndarray: + """Nimmt den eigentlichen Befehl nach dem Wake Word auf.""" + stop = threading.Event() # separater Event, läuft immer durch + audio = _record_until_silence( + stop, + max_seconds=MAX_RECORD_SECONDS, + silence_timeout=SILENCE_TIMEOUT, + ) + return audio if audio is not None else np.array([], dtype=np.float32) + + +def transcribe(audio: np.ndarray) -> str: + """Transkribiert Befehl-Audio mit dem größeren Hauptmodell.""" + if len(audio) < SAMPLE_RATE * 0.3: + return "" + return _transcribe(audio, WHISPER_MODEL_SIZE) diff --git a/client/hermes-voice/audio_output.py b/client/hermes-voice/audio_output.py new file mode 100644 index 0000000..3e2904f --- /dev/null +++ b/client/hermes-voice/audio_output.py @@ -0,0 +1,58 @@ +import asyncio +import os +import tempfile +import threading +import pygame +import edge_tts +from config import TTS_VOICE, TTS_RATE, TTS_PITCH + +pygame.mixer.init() +_stop_event = threading.Event() + + +def stop_speaking(): + """Unterbricht laufende Sprachausgabe (z.B. wenn User spricht).""" + _stop_event.set() + pygame.mixer.music.stop() + + +def speak(text: str): + """Text → Edge TTS → Lautsprecher. Blockiert bis fertig oder unterbrochen.""" + _stop_event.clear() + + async def _run(): + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f: + tmp_path = f.name + + communicate = edge_tts.Communicate(text, TTS_VOICE, rate=TTS_RATE, pitch=TTS_PITCH) + await communicate.save(tmp_path) + + if _stop_event.is_set(): + os.unlink(tmp_path) + return + + pygame.mixer.music.load(tmp_path) + pygame.mixer.music.play() + + while pygame.mixer.music.get_busy(): + if _stop_event.is_set(): + pygame.mixer.music.stop() + break + pygame.time.wait(50) + + os.unlink(tmp_path) + + asyncio.run(_run()) + + +def play_ding(): + """Kurzer Aktivierungs-Ton (440 Hz, 120ms).""" + import numpy as np + sample_rate = 44100 + duration = 0.12 + t = np.linspace(0, duration, int(sample_rate * duration), False) + wave = (np.sin(2 * np.pi * 440 * t) * 0.3 * 32767).astype(np.int16) + stereo = np.column_stack([wave, wave]) + sound = pygame.sndarray.make_sound(stereo) + sound.play() + pygame.time.wait(int(duration * 1000) + 20) diff --git a/client/hermes-voice/config.py b/client/hermes-voice/config.py new file mode 100644 index 0000000..092f4c8 --- /dev/null +++ b/client/hermes-voice/config.py @@ -0,0 +1,62 @@ +# Hermes Voice Client — Konfiguration +# Alle Einstellungen hier anpassen, kein Code-Edit nötig. + +# ── AI-Box ────────────────────────────────────────────────────────────── +MC2_BASE_URL = "http://192.168.178.151:9001/v1" + +# Modell für reine Text-Anfragen (kein Screenshot) +CHAT_MODEL = "Hermes-4-14B" + +# Modell wenn ein Screenshot mitgeschickt wird +VISION_MODEL = "Qwen3-VL-2B-Instruct" + +# ── Wake Word ──────────────────────────────────────────────────────────── +# Einfach den gewünschten Trigger-Text hier eintragen — kein Account, kein Download. +# Whisper transkribiert Sprache und prüft ob eines der Wörter enthalten ist. +# Mehrere Varianten möglich: ["hey hermes", "hermes", "hey jarvis"] +WAKE_WORDS = ["hey hermes", "hermes"] + +# Whisper-Modell für die schnelle Wake-Detection (tiny = 39MB, sehr schnell) +WAKE_WHISPER_MODEL = "tiny" + +# ── Spracheingabe ──────────────────────────────────────────────────────── +# faster-whisper Modellgröße: "tiny", "base", "small", "medium" +# "small" läuft gut auf CPU (~244 MB), "base" ist schneller aber ungenauer +WHISPER_MODEL_SIZE = "small" +WHISPER_LANGUAGE = "de" # "de" für Deutsch, "en" für Englisch, None = auto + +# Sekunden Stille bis Aufnahme endet +SILENCE_TIMEOUT = 1.5 +# Maximale Aufnahmedauer in Sekunden (Sicherheitsnetz) +MAX_RECORD_SECONDS = 20 + +# ── Sprachausgabe ──────────────────────────────────────────────────────── +# Edge TTS Stimmen: https://speech.microsoft.com/portal/voicegallery +# Deutsch: "de-DE-KillianNeural", "de-DE-SeraphinaMultilingualNeural" +# Englisch: "en-US-AndrewNeural", "en-US-AriaNeural" +TTS_VOICE = "de-DE-KillianNeural" +TTS_RATE = "+0%" # Geschwindigkeit: "+10%" schneller, "-10%" langsamer +TTS_PITCH = "+0Hz" # Tonhöhe + +# ── Screen Capture ─────────────────────────────────────────────────────── +# Screenshot bei jeder Anfrage mitschicken? +SCREENSHOT_ON_QUERY = True +# Monitor-Index (0 = alle, 1 = primär, 2 = zweiter Monitor) +SCREENSHOT_MONITOR = 1 +# Auflösung für Screenshot (kleinere = schnellere Übertragung) +SCREENSHOT_WIDTH = 1280 +SCREENSHOT_HEIGHT = 720 + +# ── Agent-Verhalten ────────────────────────────────────────────────────── +MAX_RESPONSE_TOKENS = 200 # kurze gesprochene Antworten +REQUEST_TIMEOUT = 30 # Sekunden bis Timeout + +SYSTEM_PROMPT = """Du bist Hermes, ein KI-Assistent der direkt in den lokalen AI-Homelab integriert ist. +Du hörst die Stimme des Nutzers und siehst seinen Bildschirm. + +Regeln für Antworten: +- Kurz und präzise (2–4 Sätze maximum) +- Kein Markdown, keine Aufzählungen, keine Codeblöcke — du wirst gesprochen +- Wenn du einen offensichtlichen Fehler auf dem Bildschirm siehst, weise kurz darauf hin +- Antworte auf Deutsch wenn der Nutzer Deutsch spricht, auf Englisch wenn Englisch +- Sei direkt und hilfreich, kein unnötiges Smalltalk""" diff --git a/client/hermes-voice/main.py b/client/hermes-voice/main.py new file mode 100644 index 0000000..addd836 --- /dev/null +++ b/client/hermes-voice/main.py @@ -0,0 +1,129 @@ +""" +Hermes Voice Client +Wake Word → STT → Screenshot → Hermes (MC2) → TTS + +Läuft als System-Tray-App im Hintergrund. +Aktivierung: "Hey Jarvis" (konfigurierbar in config.py) +""" + +import threading +import sys +from PIL import Image, ImageDraw +import pystray + +from audio_input import wait_for_wake_word, record_speech, transcribe +from audio_output import speak, play_ding, stop_speaking +from screen_capture import capture_screen +from ai_client import ask +from config import SCREENSHOT_ON_QUERY + +# ── Status ─────────────────────────────────────────────────────────────── +class State: + IDLE = "idle" # wartet auf Wake Word + LISTENING = "listen" # nimmt Sprache auf + THINKING = "think" # wartet auf MC2-Antwort + SPEAKING = "speak" # gibt Antwort aus + +_state = State.IDLE +_stop_event = threading.Event() +_tray_icon: pystray.Icon | None = None + + +# ── Tray Icon ──────────────────────────────────────────────────────────── +COLORS = { + State.IDLE: "#22c55e", # grün + State.LISTENING: "#eab308", # gelb + State.THINKING: "#3b82f6", # blau + State.SPEAKING: "#a855f7", # lila +} + +def _make_icon(color: str) -> Image.Image: + img = Image.new("RGBA", (64, 64), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + draw.ellipse([4, 4, 60, 60], fill=color) + return img + +def _set_state(state: str): + global _state + _state = state + if _tray_icon: + _tray_icon.icon = _make_icon(COLORS[state]) + labels = { + State.IDLE: "Hermes — bereit (Hey Jarvis)", + State.LISTENING: "Hermes — hört zu…", + State.THINKING: "Hermes — denkt…", + State.SPEAKING: "Hermes — spricht", + } + _tray_icon.title = labels[state] + + +# ── Hauptloop ──────────────────────────────────────────────────────────── +def _voice_loop(): + print("[Hermes] Bereit. Sage 'Hey Jarvis' zum Aktivieren.") + while not _stop_event.is_set(): + _set_state(State.IDLE) + + # 1. Auf Wake Word warten + detected = wait_for_wake_word(_stop_event) + if not detected: + break + + # 2. Aktivierungs-Ding + Aufnahme starten + play_ding() + _set_state(State.LISTENING) + audio = record_speech() + + # 3. Transkribieren + text = transcribe(audio) + if not text: + continue + print(f"[STT] {text!r}") + + # 4. Screenshot machen + screenshot = capture_screen() if SCREENSHOT_ON_QUERY else None + + # 5. MC2 fragen + _set_state(State.THINKING) + response = ask(text, screenshot) + print(f"[Hermes] {response!r}") + + # 6. Antwort sprechen + _set_state(State.SPEAKING) + speak(response) + + print("[Hermes] Beendet.") + + +# ── System Tray ────────────────────────────────────────────────────────── +def _on_quit(icon, item): + _stop_event.set() + stop_speaking() + icon.stop() + +def _on_mute(icon, item): + stop_speaking() + +def main(): + global _tray_icon + + loop_thread = threading.Thread(target=_voice_loop, daemon=True) + loop_thread.start() + + menu = pystray.Menu( + pystray.MenuItem("Hermes Voice", None, enabled=False), + pystray.Menu.SEPARATOR, + pystray.MenuItem("Sprechen stoppen", _on_mute), + pystray.MenuItem("Beenden", _on_quit), + ) + + _tray_icon = pystray.Icon( + "hermes-voice", + icon=_make_icon(COLORS[State.IDLE]), + title="Hermes — bereit (Hey Jarvis)", + menu=menu, + ) + _tray_icon.run() + + +if __name__ == "__main__": + main() diff --git a/client/hermes-voice/requirements.txt b/client/hermes-voice/requirements.txt new file mode 100644 index 0000000..848190e --- /dev/null +++ b/client/hermes-voice/requirements.txt @@ -0,0 +1,28 @@ +# Hermes Voice Client — Dependencies +# Installation: pip install -r requirements.txt + +# Wake Word Detection: Silero VAD + Whisper (kein Account, beliebiges Wort) + +# Speech-to-Text (läuft lokal auf CPU) +faster-whisper>=1.0.0 + +# Audio I/O +sounddevice>=0.4.6 +numpy>=1.24.0 +scipy>=1.11.0 + +# Text-to-Speech +edge-tts>=6.1.9 + +# Screen Capture +mss>=9.0.1 +Pillow>=10.0.0 + +# HTTP Client (async) +httpx>=0.27.0 + +# System Tray +pystray>=0.19.5 + +# Audio Playback +pygame>=2.5.0 diff --git a/client/hermes-voice/screen_capture.py b/client/hermes-voice/screen_capture.py new file mode 100644 index 0000000..22f6be5 --- /dev/null +++ b/client/hermes-voice/screen_capture.py @@ -0,0 +1,19 @@ +import base64 +import io +import mss +from PIL import Image +from config import SCREENSHOT_MONITOR, SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT + + +def capture_screen() -> str: + """Screenshot des primären Monitors als base64-JPEG.""" + with mss.mss() as sct: + monitor = sct.monitors[SCREENSHOT_MONITOR] + raw = sct.grab(monitor) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + + img = img.resize((SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT), Image.LANCZOS) + + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=80) + return base64.b64encode(buf.getvalue()).decode("utf-8") diff --git a/client/hermes-voice/setup.bat b/client/hermes-voice/setup.bat new file mode 100644 index 0000000..35ff47d --- /dev/null +++ b/client/hermes-voice/setup.bat @@ -0,0 +1,40 @@ +@echo off +echo ======================================== +echo Hermes Voice Client — Setup +echo ======================================== +echo. + +:: Python pruefen +python --version >nul 2>&1 +if errorlevel 1 ( + echo [FEHLER] Python nicht gefunden. Bitte Python 3.11+ installieren. + pause + exit /b 1 +) + +:: Venv anlegen falls nicht vorhanden +if not exist ".venv" ( + echo [1/3] Erstelle virtuelle Umgebung... + python -m venv .venv +) + +:: Dependencies installieren +echo [2/3] Installiere Dependencies... +.venv\Scripts\pip install --upgrade pip -q +.venv\Scripts\pip install -r requirements.txt + +echo [3/3] Pruefe Porcupine-Konfiguration... +.venv\Scripts\python -c "from config import PORCUPINE_ACCESS_KEY; print('OK' if PORCUPINE_ACCESS_KEY != 'DEIN_ACCESS_KEY_HIER' else 'WARNUNG: Access Key fehlt noch in config.py!')" + +echo. +echo ======================================== +echo Setup abgeschlossen! +echo. +echo Naechste Schritte: +echo 1. https://console.picovoice.ai/ - kostenlos registrieren +echo 2. Access Key in config.py eintragen (PORCUPINE_ACCESS_KEY) +echo 3. Wake Word trainieren - .ppn Datei in diesen Ordner legen +echo 4. Dateiname in config.py eintragen (WAKE_WORD_MODEL_PATH) +echo 5. start.bat +echo ======================================== +pause diff --git a/client/hermes-voice/start.bat b/client/hermes-voice/start.bat new file mode 100644 index 0000000..62f6d0f --- /dev/null +++ b/client/hermes-voice/start.bat @@ -0,0 +1,4 @@ +@echo off +:: Hermes Voice Client starten (kein Konsolenfenster im Hintergrund) +cd /d "%~dp0" +start "" .venv\Scripts\pythonw.exe main.py