From 45fb61000a62b33f642ee4dac830bf1724f0ed37 Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Thu, 2 Jul 2026 10:49:36 +0200 Subject: [PATCH] =?UTF-8?q?VAD:=20Silero=20v5=20(vad-web)=20statt=20RMS-Ei?= =?UTF-8?q?genbau=20=E2=80=94=20Turn-Ende=20900=20->=20450=20ms=20(Review?= =?UTF-8?q?=20P1-7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - useVAD.ts: MicVAD (neuronale Sprach- statt Pegel-Erkennung), redemptionMs 450, preSpeechPad 320 ms, WAV direkt aus Float32 (kein webm/Opus-Umweg mehr zur Box) - boxStt.ts: Dateiname passend zum Blob-Typ (rec.wav | rec.webm) - Assets self-hosted unter /vad/ via scripts/copy-vad-assets.mjs (npm postinstall), offlinefaehig - Verifiziert: tsc + Build sauber, App startet, Assets werden ausgeliefert (Mikro-Test: User) Co-Authored-By: Claude Fable 5 --- client/lucy-desktop/.gitignore | 1 + client/lucy-desktop/electron.vite.config.ts | 3 + client/lucy-desktop/package-lock.json | 137 ++++++++++++++++- client/lucy-desktop/package.json | 4 +- .../lucy-desktop/scripts/copy-vad-assets.mjs | 23 +++ .../src/renderer/src/lib/voice/useVAD.ts | 144 ++++++++---------- .../renderer/src/voice-core/engines/boxStt.ts | 8 +- 7 files changed, 230 insertions(+), 90 deletions(-) create mode 100644 client/lucy-desktop/scripts/copy-vad-assets.mjs diff --git a/client/lucy-desktop/.gitignore b/client/lucy-desktop/.gitignore index d6530a4..f26aa03 100644 --- a/client/lucy-desktop/.gitignore +++ b/client/lucy-desktop/.gitignore @@ -4,3 +4,4 @@ dist/ *.log # Avatar-VRM ist gross + Lizenz/Redistribution -> nicht committen (lokal/Box vorhalten) src/renderer/public/avatar.vrm +src/renderer/public/vad/ diff --git a/client/lucy-desktop/electron.vite.config.ts b/client/lucy-desktop/electron.vite.config.ts index a620993..0e79038 100644 --- a/client/lucy-desktop/electron.vite.config.ts +++ b/client/lucy-desktop/electron.vite.config.ts @@ -12,6 +12,9 @@ export default defineConfig({ renderer: { root: resolve(__dirname, "src/renderer"), build: { rollupOptions: { input: { index: resolve(__dirname, "src/renderer/index.html") } } }, + // Silero-VAD-Assets (Worklet + ONNX + onnxruntime-WASM) liegen self-hosted in + // src/renderer/public/vad/ (scripts/copy-vad-assets.mjs, läuft via npm postinstall) — + // Pfade = baseAssetPath/onnxWASMBasePath in useVAD.ts. plugins: [react()], }, }) diff --git a/client/lucy-desktop/package-lock.json b/client/lucy-desktop/package-lock.json index a6835d5..d3d1cab 100644 --- a/client/lucy-desktop/package-lock.json +++ b/client/lucy-desktop/package-lock.json @@ -7,11 +7,13 @@ "": { "name": "lucy-desktop", "version": "0.1.0", + "hasInstallScript": true, "dependencies": { "@pixiv/three-vrm": "^3.4.0", "@pixiv/three-vrm-animation": "^3.5.4", "@react-three/drei": "^9.114.0", "@react-three/fiber": "^8.17.10", + "@ricky0123/vad-web": "^0.0.30", "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.1.0", @@ -1027,6 +1029,63 @@ "@pixiv/types-vrmc-vrm-1.0": "3.5.4" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, "node_modules/@react-spring/animated": { "version": "9.7.5", "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.5.tgz", @@ -1207,6 +1266,15 @@ } } }, + "node_modules/@ricky0123/vad-web": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@ricky0123/vad-web/-/vad-web-0.0.30.tgz", + "integrity": "sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==", + "license": "ISC", + "dependencies": { + "onnxruntime-web": "^1.17.0" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.27", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", @@ -1768,7 +1836,6 @@ "version": "20.19.43", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -2630,6 +2697,12 @@ "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", "license": "MIT" }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -2790,6 +2863,12 @@ "dev": true, "license": "ISC" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -3078,6 +3157,12 @@ "immediate": "~3.0.5" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -4112,6 +4197,26 @@ "wrappy": "1" } }, + "node_modules/onnxruntime-common": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.27.0.tgz", + "integrity": "sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==", + "license": "MIT" + }, + "node_modules/onnxruntime-web": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.27.0.tgz", + "integrity": "sha512-ogDLsqIozHZwifPuN37OproAo0byX6t43/bP8GzeZWBWD6MOGExswFAx3up4NS/vvWBOg2u2PXomDt3rMmdQSg==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.27.0", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, "node_modules/p-cancelable": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", @@ -4183,6 +4288,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/postcss": { "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", @@ -4259,6 +4370,29 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", @@ -4910,7 +5044,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/unified": { diff --git a/client/lucy-desktop/package.json b/client/lucy-desktop/package.json index 5e1f32e..25fa1b4 100644 --- a/client/lucy-desktop/package.json +++ b/client/lucy-desktop/package.json @@ -7,13 +7,15 @@ "scripts": { "dev": "electron-vite dev", "build": "electron-vite build", - "preview": "electron-vite preview" + "preview": "electron-vite preview", + "postinstall": "node scripts/copy-vad-assets.mjs" }, "dependencies": { "@pixiv/three-vrm": "^3.4.0", "@pixiv/three-vrm-animation": "^3.5.4", "@react-three/drei": "^9.114.0", "@react-three/fiber": "^8.17.10", + "@ricky0123/vad-web": "^0.0.30", "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.1.0", diff --git a/client/lucy-desktop/scripts/copy-vad-assets.mjs b/client/lucy-desktop/scripts/copy-vad-assets.mjs new file mode 100644 index 0000000..3b4df3d --- /dev/null +++ b/client/lucy-desktop/scripts/copy-vad-assets.mjs @@ -0,0 +1,23 @@ +// Kopiert die Silero-VAD-Assets (Worklet + ONNX-Modell + onnxruntime-WASM) nach public/vad/, +// damit die App sie self-hosted lädt (offlinefähig, kein CDN). Läuft via npm postinstall. +// public/ wird von Vite in Dev UND Build verbatim ausgeliefert -> ein Pfad für beide Welten. +import { copyFileSync, mkdirSync, readdirSync } from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" + +const root = join(dirname(fileURLToPath(import.meta.url)), "..") +const dest = join(root, "src", "renderer", "public", "vad") +mkdirSync(dest, { recursive: true }) + +const jobs = [ + [join(root, "node_modules", "@ricky0123", "vad-web", "dist"), (f) => f === "vad.worklet.bundle.min.js" || f.endsWith(".onnx")], + [join(root, "node_modules", "onnxruntime-web", "dist"), (f) => f.endsWith(".wasm") || f.endsWith(".mjs")], +] + +let n = 0 +for (const [src, match] of jobs) { + for (const f of readdirSync(src)) { + if (match(f)) { copyFileSync(join(src, f), join(dest, f)); n++ } + } +} +console.log(`[vad-assets] ${n} Dateien nach ${dest} kopiert.`) diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts b/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts index 27fca5d..f77653d 100644 --- a/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts +++ b/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts @@ -1,8 +1,12 @@ import { useEffect, useRef } from "react" +import { MicVAD } from "@ricky0123/vad-web" -// Freisprech-VAD (Voice Activity Detection): lauscht dauerhaft am Mikro, erkennt Sprech-Beginn/-Ende -// per Energie und liefert komplette Äußerungen als Blob. RÜCKKOPPLUNGS-SCHUTZ: `paused` (Lucy denkt/spricht) -// stoppt die Erkennung -> Lucys eigene Stimme triggert das Mikro nicht; + echoCancellation + Abkling-Sperre. +// Freisprech-VAD: lauscht dauerhaft am Mikro und liefert komplette Äußerungen als Blob. +// Seit Review P1-7 (2026-07-02) Silero VAD v5 (neuronal, via vad-web/onnxruntime-wasm) statt +// des RMS-Eigenbaus: erkennt Sprache statt Lautstärke (robust gegen Tastatur/Lüfter) und +// erlaubt dadurch ein kürzeres Turn-Ende (450 ms statt 900 ms Stille) ohne Abschneiden. +// RÜCKKOPPLUNGS-SCHUTZ bleibt: `paused` (Lucy denkt/spricht) verwirft laufende Erkennung, +// + echoCancellation + Abkling-Sperre nach Lucys Antwort. interface VADOptions { enabled: boolean // VAD-Modus an? @@ -11,11 +15,28 @@ interface VADOptions { onListening?: (active: boolean) => void // UI: gerade Sprache am Aufnehmen? } -// Erkennung per ZEIT-DOMÄNEN-RMS mit ADAPTIVEM Rausch-Pegel (kalibriert sich aufs Mikro) — robuster -// als feste Frequenz-Schwellen. Schwellen werden relativ zum gemessenen Rauschen berechnet. -const START_FRAMES = 3 // so viele laute Frames in Folge -> Start (gegen Klick-Fehlstarts) -const SILENCE_MS = 900 // so lange Stille -> Äußerung zu Ende -const COOLDOWN_MS = 450 // nach Pause-Ende kurz taub (Echo/Lautsprecher abklingen lassen) +const REDEMPTION_MS = 450 // so lange Stille -> Äußerung zu Ende (war 900 ms beim RMS-VAD) +const PRE_SPEECH_PAD_MS = 320 // Vorlauf mitschneiden (erster Wortanfang nicht abschneiden) +const MIN_SPEECH_MS = 160 // kürzer = Klick/Räuspern -> verwerfen +const COOLDOWN_MS = 450 // nach Lucys Antwort kurz taub (Echo/Lautsprecher abklingen lassen) + +// Float32-Samples (16 kHz mono) -> WAV-Blob (PCM16). Ersetzt den MediaRecorder-webm-Umweg: +// die Box muss kein Opus mehr dekodieren, Parakeet/Whisper bekommen direkt sauberes WAV. +function toWavBlob(samples: Float32Array, sampleRate = 16000): Blob { + const buf = new ArrayBuffer(44 + samples.length * 2) + const v = new DataView(buf) + const writeStr = (off: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(off + i, s.charCodeAt(i)) } + writeStr(0, "RIFF"); v.setUint32(4, 36 + samples.length * 2, true); writeStr(8, "WAVE") + writeStr(12, "fmt "); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true) + v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true) + writeStr(36, "data"); v.setUint32(40, samples.length * 2, true) + let off = 44 + for (let i = 0; i < samples.length; i++, off += 2) { + const s = Math.max(-1, Math.min(1, samples[i])) + v.setInt16(off, s < 0 ? s * 0x8000 : s * 0x7fff, true) + } + return new Blob([buf], { type: "audio/wav" }) +} export function useVAD({ enabled, paused, onUtterance, onListening }: VADOptions) { const pausedRef = useRef(paused); pausedRef.current = paused @@ -29,92 +50,47 @@ export function useVAD({ enabled, paused, onUtterance, onListening }: VADOptions useEffect(() => { if (!enabled) return let cancelled = false - let stream: MediaStream | null = null - let ctx: AudioContext | null = null - let raf = 0 - let rec: MediaRecorder | null = null - let chunks: Blob[] = [] - let speaking = false - let silenceStart = 0 - let voiceFrames = 0 - - const setListening = (v: boolean) => onLst.current?.(v) + let vad: Awaited> | null = null ;(async () => { try { - stream = await navigator.mediaDevices.getUserMedia({ - audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + vad = await MicVAD.new({ + model: "v5", + // Assets self-hosted (electron.vite.config.ts kopiert sie nach /vad/) — offlinefähig. + baseAssetPath: "/vad/", + onnxWASMBasePath: "/vad/", + positiveSpeechThreshold: 0.5, + negativeSpeechThreshold: 0.35, + redemptionMs: REDEMPTION_MS, + preSpeechPadMs: PRE_SPEECH_PAD_MS, + minSpeechMs: MIN_SPEECH_MS, + getStream: () => navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }), + onSpeechStart: () => { + if (pausedRef.current || performance.now() < resumeAt.current) return + onLst.current?.(true) + }, + onSpeechEnd: (audio: Float32Array) => { + onLst.current?.(false) + // Während Lucy denkt/spricht (oder direkt danach) erkannte Sprache = ihr eigenes + // Echo bzw. Nachhall -> verwerfen statt transkribieren. + if (cancelled || pausedRef.current || performance.now() < resumeAt.current) return + if (audio.length < MIN_SPEECH_MS * 16) return // 16 Samples/ms @16 kHz + onUtt.current(toWavBlob(audio)) + }, + onVADMisfire: () => onLst.current?.(false), }) + if (cancelled) { vad.destroy(); return } + vad.start() } catch (e) { - console.error("VAD: Mikrofon-Zugriff verweigert", e); return + console.error("VAD: Silero-Init fehlgeschlagen", e) } - if (cancelled) { stream.getTracks().forEach((t) => t.stop()); return } - - const Ctor = window.AudioContext || (window as any).webkitAudioContext - ctx = new Ctor() - try { await ctx.resume() } catch { /* */ } // WICHTIG: sonst bleibt der Context 'suspended' -> nur Nullen - const srcNode = ctx.createMediaStreamSource(stream) - const analyser = ctx.createAnalyser() - analyser.fftSize = 1024 - srcNode.connect(analyser) - const wave = new Uint8Array(analyser.fftSize) - let noiseFloor = 0.015 // adaptiver Rausch-Pegel (kalibriert sich auf das Mikro) - const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/webm" - - const startRec = () => { - chunks = [] - rec = new MediaRecorder(stream!, { mimeType: mime }) - rec.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data) } - rec.onstop = () => { - const blob = new Blob(chunks, { type: mime }) - if (!cancelled && blob.size > 2400) onUtt.current(blob) // Mini-Blobs (Klicks) verwerfen - } - rec.start() - setListening(true) - } - const stopRec = (emit: boolean) => { - if (rec && rec.state !== "inactive") { - if (!emit) rec.onstop = null - rec.stop() - } - rec = null - setListening(false) - } - - const tick = () => { - if (cancelled) return - analyser.getByteTimeDomainData(wave) - let sq = 0 - for (let i = 0; i < wave.length; i++) { const v = (wave[i] - 128) / 128; sq += v * v } - const level = Math.sqrt(sq / wave.length) // RMS-Pegel 0..1 - const now = performance.now() - // Rausch-Pegel langsam an leise Phasen angleichen (nur wenn nicht geredet wird) - if (!speaking) noiseFloor += (Math.min(level, noiseFloor * 1.6 + 0.004) - noiseFloor) * 0.05 - const startTh = Math.max(0.02, noiseFloor * 2.8) // Sprache: deutlich über dem Rauschen - const stopTh = Math.max(0.012, noiseFloor * 1.7) // Stille: nahe am Rauschen - - if (pausedRef.current || now < resumeAt.current) { - if (speaking) { speaking = false; voiceFrames = 0; silenceStart = 0; stopRec(false) } // verwerfen - } else if (!speaking) { - if (level > startTh) { voiceFrames++; if (voiceFrames >= START_FRAMES) { speaking = true; silenceStart = 0; startRec() } } - else voiceFrames = 0 - } else { - if (level < stopTh) { - if (!silenceStart) silenceStart = now - else if (now - silenceStart > SILENCE_MS) { speaking = false; silenceStart = 0; stopRec(true) } - } else silenceStart = 0 - } - raf = requestAnimationFrame(tick) - } - tick() })() return () => { cancelled = true - cancelAnimationFrame(raf) - try { if (rec && rec.state !== "inactive") { rec.onstop = null; rec.stop() } } catch { /* */ } - stream?.getTracks().forEach((t) => t.stop()) - try { ctx?.close() } catch { /* */ } + try { vad?.destroy() } catch { /* */ } onLst.current?.(false) } }, [enabled]) diff --git a/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts b/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts index e117fea..700f659 100644 --- a/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts +++ b/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts @@ -1,14 +1,16 @@ -// Box-STT-Adapter — Spracherkennung läuft auf der Box (MC2 /api/voice/stt -> faster-whisper). +// Box-STT-Adapter — Spracherkennung läuft auf der Box (MC2 /api/voice/stt -> Parakeet, +// whisper als Fallback). Der Dateiname muss zum Blob-Typ passen: der Sidecar wählt die +// Dekodierung nach Suffix (Silero-VAD liefert WAV, Push-to-Talk weiterhin webm/opus). import { BOX_URL } from "../../config" import type { SttEngine } from "../types" export class BoxSttEngine implements SttEngine { - readonly id = "box-whisper" + readonly id = "box-stt" constructor(private readonly baseUrl: string = BOX_URL) {} async transcribe(audio: Blob): Promise { const fd = new FormData() - fd.append("audio", audio, "rec.webm") + fd.append("audio", audio, audio.type.includes("wav") ? "rec.wav" : "rec.webm") const r = await fetch(`${this.baseUrl}/api/voice/stt`, { method: "POST", body: fd }) if (!r.ok) throw new Error(`STT ${r.status}`) return (await r.json()).text?.trim() || ""