From aff01057006bb02744c5932856b4d6707e414961 Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Thu, 2 Jul 2026 10:29:22 +0200 Subject: [PATCH] Lucy v2: voice-core-Adapter, VAD, Overlay-Feinschliff + Projekt-Dateien - voice-core/: STT/TTS hinter Engine-Interfaces (pocket/box austauschbar), SpeechScheduler ausgelagert - useVAD (Freisprechen, adaptive RMS-Schwelle), usePushToTalk, sentiment - Avatar3D/config/styles-Feinschliff, AuraGlow - electron.vite.config, tsconfig, Starter-BATs, .gitignore (out/, avatar.vrm) Co-Authored-By: Claude Fable 5 --- client/lucy-desktop/.gitignore | 6 + client/lucy-desktop/Lucy-Check.bat | 8 + client/lucy-desktop/Lucy-Dev-Debug.bat | 13 ++ client/lucy-desktop/Lucy-Neustart.bat | 12 ++ .../lucy-desktop/Verknuepfung-erstellen.bat | 7 + client/lucy-desktop/electron.vite.config.ts | 17 ++ client/lucy-desktop/lucy.ico | Bin 0 -> 16608 bytes client/lucy-desktop/src/renderer/index.html | 13 ++ .../src/renderer/src/components/AuraGlow.tsx | 47 +++++ .../src/renderer/src/components/Avatar3D.tsx | 57 +++++- .../lucy-desktop/src/renderer/src/config.ts | 19 +- .../src/renderer/src/lib/voice/sentiment.ts | 14 ++ .../renderer/src/lib/voice/usePushToTalk.ts | 45 +++++ .../src/renderer/src/lib/voice/useVAD.ts | 121 +++++++++++++ .../renderer/src/lib/voice/useVoiceAgent.ts | 167 ++++++++++++------ client/lucy-desktop/src/renderer/src/main.tsx | 10 ++ .../lucy-desktop/src/renderer/src/styles.css | 26 +++ .../renderer/src/voice-core/engines/boxStt.ts | 16 ++ .../src/voice-core/engines/pocketTts.ts | 39 ++++ .../src/renderer/src/voice-core/index.ts | 14 ++ .../src/renderer/src/voice-core/speech.ts | 84 +++++++++ .../src/renderer/src/voice-core/types.ts | 27 +++ client/lucy-desktop/start-lucy.cmd | 9 + client/lucy-desktop/tsconfig.json | 15 ++ 24 files changed, 727 insertions(+), 59 deletions(-) create mode 100644 client/lucy-desktop/.gitignore create mode 100644 client/lucy-desktop/Lucy-Check.bat create mode 100644 client/lucy-desktop/Lucy-Dev-Debug.bat create mode 100644 client/lucy-desktop/Lucy-Neustart.bat create mode 100644 client/lucy-desktop/Verknuepfung-erstellen.bat create mode 100644 client/lucy-desktop/electron.vite.config.ts create mode 100644 client/lucy-desktop/lucy.ico create mode 100644 client/lucy-desktop/src/renderer/index.html create mode 100644 client/lucy-desktop/src/renderer/src/components/AuraGlow.tsx create mode 100644 client/lucy-desktop/src/renderer/src/lib/voice/sentiment.ts create mode 100644 client/lucy-desktop/src/renderer/src/lib/voice/usePushToTalk.ts create mode 100644 client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts create mode 100644 client/lucy-desktop/src/renderer/src/main.tsx create mode 100644 client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts create mode 100644 client/lucy-desktop/src/renderer/src/voice-core/engines/pocketTts.ts create mode 100644 client/lucy-desktop/src/renderer/src/voice-core/index.ts create mode 100644 client/lucy-desktop/src/renderer/src/voice-core/speech.ts create mode 100644 client/lucy-desktop/src/renderer/src/voice-core/types.ts create mode 100644 client/lucy-desktop/start-lucy.cmd create mode 100644 client/lucy-desktop/tsconfig.json diff --git a/client/lucy-desktop/.gitignore b/client/lucy-desktop/.gitignore new file mode 100644 index 0000000..d6530a4 --- /dev/null +++ b/client/lucy-desktop/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +out/ +dist/ +*.log +# Avatar-VRM ist gross + Lizenz/Redistribution -> nicht committen (lokal/Box vorhalten) +src/renderer/public/avatar.vrm diff --git a/client/lucy-desktop/Lucy-Check.bat b/client/lucy-desktop/Lucy-Check.bat new file mode 100644 index 0000000..ccdbf37 --- /dev/null +++ b/client/lucy-desktop/Lucy-Check.bat @@ -0,0 +1,8 @@ +@echo off +chcp 65001 >nul +cd /d "%~dp0" +curl -s -m 5 http://127.0.0.1:8130/health > lucy_check.log 2>&1 +echo.>> lucy_check.log +echo --- electron/vite/pocket laeuft? --->> lucy_check.log +powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'pocket_server:app|electron-vite|lucy-desktop' } | ForEach-Object { $_.Name + ' ' + $_.ProcessId }" >> lucy_check.log 2>&1 +type lucy_check.log diff --git a/client/lucy-desktop/Lucy-Dev-Debug.bat b/client/lucy-desktop/Lucy-Dev-Debug.bat new file mode 100644 index 0000000..58233aa --- /dev/null +++ b/client/lucy-desktop/Lucy-Dev-Debug.bat @@ -0,0 +1,13 @@ +@echo off +chcp 65001 >nul +cd /d "F:\Coding Stuff\mission-control-2\client\lucy-desktop" +echo Beende haengende Versuche (ohne mich selbst)... +powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.Name -eq 'electron.exe' -or $_.CommandLine -match 'electron-vite|pocket_server:app|start-lucy' } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -EA SilentlyContinue }" +timeout /t 2 >nul +echo Starte 'npm run dev' mit Log-Capture... +start /b "" cmd /c "npm run dev > dev.log 2>&1" +timeout /t 22 >nul +echo === dev.log === +type dev.log +echo. +echo (laeuft im Hintergrund weiter, falls erfolgreich) diff --git a/client/lucy-desktop/Lucy-Neustart.bat b/client/lucy-desktop/Lucy-Neustart.bat new file mode 100644 index 0000000..af9d02e --- /dev/null +++ b/client/lucy-desktop/Lucy-Neustart.bat @@ -0,0 +1,12 @@ +@echo off +chcp 65001 >nul +REM Hygiene: Recon-Log mit Token entfernen +del /q "F:\Coding Stuff\mission-control-2\box_recon.log" 2>nul +echo Beende laufende Lucy-Instanz (pocket_server :8130 + Electron/Vite)... +powershell -NoProfile -ExecutionPolicy Bypass -Command "Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'pocket_server:app|lucy-desktop|electron-vite' } | ForEach-Object { Write-Output ('kill ' + $_.ProcessId + ' ' + $_.Name); Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }" +timeout /t 2 >nul +echo. +echo Starte Lucy neu (frischer Voice-Server + Client mit erweitertem Umlaut-Fix)... +start "" "F:\Coding Stuff\mission-control-2\client\lucy-desktop\start-lucy.cmd" +timeout /t 3 >nul +echo Fertig. diff --git a/client/lucy-desktop/Verknuepfung-erstellen.bat b/client/lucy-desktop/Verknuepfung-erstellen.bat new file mode 100644 index 0000000..c5aed20 --- /dev/null +++ b/client/lucy-desktop/Verknuepfung-erstellen.bat @@ -0,0 +1,7 @@ +@echo off +chcp 65001 >nul +powershell -NoProfile -ExecutionPolicy Bypass -Command "$ws=New-Object -ComObject WScript.Shell; $p=Join-Path ([Environment]::GetFolderPath('Desktop')) 'Lucy.lnk'; $l=$ws.CreateShortcut($p); $l.TargetPath='F:\Coding Stuff\mission-control-2\client\lucy-desktop\start-lucy.cmd'; $l.WorkingDirectory='F:\Coding Stuff\mission-control-2\client\lucy-desktop'; $l.IconLocation='F:\Coding Stuff\mission-control-2\client\lucy-desktop\lucy.ico'; $l.Description='Lucy - lokaler Sprach-Companion starten'; $l.Save(); Write-Output ('OK: ' + $p)" > lucy_shortcut.log 2>&1 +type lucy_shortcut.log +echo. +echo Fertig - "Lucy" liegt jetzt auf dem Desktop. +pause diff --git a/client/lucy-desktop/electron.vite.config.ts b/client/lucy-desktop/electron.vite.config.ts new file mode 100644 index 0000000..a620993 --- /dev/null +++ b/client/lucy-desktop/electron.vite.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "electron-vite" +import react from "@vitejs/plugin-react" +import { resolve } from "path" + +export default defineConfig({ + main: { + build: { rollupOptions: { input: { index: resolve(__dirname, "src/main/index.ts") } } }, + }, + preload: { + build: { rollupOptions: { input: { index: resolve(__dirname, "src/preload/index.ts") } } }, + }, + renderer: { + root: resolve(__dirname, "src/renderer"), + build: { rollupOptions: { input: { index: resolve(__dirname, "src/renderer/index.html") } } }, + plugins: [react()], + }, +}) diff --git a/client/lucy-desktop/lucy.ico b/client/lucy-desktop/lucy.ico new file mode 100644 index 0000000000000000000000000000000000000000..f5849fe7fd99fdbf4e072111883606be83f48b8c GIT binary patch literal 16608 zcmZ{M3p`Zm|Npp!R6~*^CaJYuCrOgbR93bX)2_<4VY*TwNvO=ltyGfQwiL@~+epe3 za!FUX!!PjP!hdvTfDc}$ANs;yh5X{O4VCkDQ6Kn1({HQW zRut-4qMrPq7Wi8C)Q|f@P^dW-@E^76pod3Ls4c`DKWyE7)@P_~-yO3Lp;*!8?qOWF zb>g{Er_(VZ!q7$4!T)TDJ{)5mbHifip}7aF(T8#uXw5yAwq@i}eZbOt+Yc9R*|vCn zSfT!XYhV35)Xi22C-jlKC;hqjH-F(m_t5Ys!u;i+q>b5uVuw$Uz#%2Tc zKpJ^iFYhzIu-9+J;~`dVXyv^fGu&n;w1$++D+TSnU;ZNSpODA@*!NCx?y9`Qj~XP{ zCoV8?R5Wj~YR>c4BsCJu10)abGRVr{5LK>SU1NeU#n}OO89ir1a2eI*tcz9W(Q@69 zBvE+#ZVxl8>At`-l^8Eofd~0@4!gE%pxv%;)!7O!`}<8^_KIIiLRhr(nlX}C^e|ES z%KssDhxElhmpUi*%fRj+S#!X-A=A;I*~cT^mYE+(Svk`!G=DI^&@XQ!XJ9pCB?CmdwG-yd-qJ_pmpB0WNlBz(fNV+;|24BbM9e(3m`lyXS#W~ z-P*h1;&)b~(Qo;jWAfT~VsaLnd`#dapFcyWOMHcX=FiuV$M6lIWl(`CqBHwp(4El6 z&AI)l>T`77Rdsu9^?0z?6ekt+wYrbHe0=GbliC)PliXr#w~cs&8`rGCx0I`SVE)21 zCNdALahJZ|K~LtbiKtlBHm)me>vQNoF-OzQ=(i1rF6IdB%%G~B?NF47Y;+)S)E&yn z66z5RBnHxT{5(jTj}|C`iH>9G=b*cVbmLNK|i9b5o<~b>vw{T@gMu+TNSy*l^AE{&+BFX`A#M`@MEPTHPaQR?k!CRr`d1f@{*)*-6Q zcqXl4;sGA5$joG*F#H}q$NZMqGNP_Bu{7Yj$LDK=Tk0`p$8T#{<`2HZGYG_Ut9k8@ zZ4TE|7lZd~c1iYr6Y&BjioJdtjfGLC>MQ4yE>RmTvCn0l)#6I+=sIo2cB+#n$FNkK zu?8|Ze&Wt;$A14VhU5L94x*}Yp!?B2jV9qi%Ted@o=(PGTscOaR*7bXh{#=xS@gMm-z#(tKl67>/_<(nB_(iEXs+Pd&H=Sp@%ODLDT zH8A|7PKGAFDzVIO;XThX$yM20`2tnNdSwx&^)O|u-aO>w-e;aCF6r*@~jlxDkH`v%%P?-ZRkM}?;^Xy1?-;smHdb*vM%Kq zH=GqD(P#_i*$VHR-lrdG>l(e#^R!1|kx^5I2Jd3(r|fr|Fdf+Q{g$I9avg@SQF2MP ztehZ|Yrm*_kSMfWS%P<_88wpMz3%S5xA1n`@2-a9LmCdlztOe`V};gZ`ylQnJS(x@ zb2cd4ijI;DoEqMb>(Xl`;y6;mxi|MR6L2BnyPNjr6&=DAmiI#C09cdAa*Tcw6$Hwfvc{VMOG42C$7> zl*QEsBrD}YmJ8&>4}lD%vnZN8J3++bk>VosGwm`L=yDdXO2zSxIp5h|SnoG8JfCT= zv-{WAb&i9pOT9k1^s4=@8YH6Z7BSr9`-3jK&QhefXv)wkHJ#S{}Bnr<1alGz0%CCkibxR#S0h;RUka zU5M)k#FQwUzi4rNz|TI1Doeh!#@W_mDd{vkw7T_gjVSGjbz-l+z*>~?k2TZW{+MgO`wyWaL9$dD>tysE2?;H`|NDS^ z@BvQtVzp*k41u4-*LG7}>1gmEUXrbVq(zunv~)QY`*RztiAcQY7t)?5T(RIB(-f~SHfRh|6#MU&QIt64tJLHbbKA9ILEP&S7ZZR6@Fe2?}BKDJSrG7R+Ll3c9%vi zBlp_Onxk%dFUny5u&;Tq@%no4cQk+BWqvaMd6EaAV1KI9j>qDxb@A>xpm_SzgDGwsdW{4mkQBD$vObtZ)4}9FF6{D6iX&fXVDIg!VyK zLUEUwKfxdD$M%3r;i7QIHGN{3zM`6BX$!7KoXSpF#=-wU8w%ri{+`fS6T{uX%yHt~ zP3bPpiDP%fHQS2RD9Z&sF~Vf2I*DCGk74tX7nQ$sl!(IBDbYN!eR%lY&m~-fL2J0@ z)uNlx!wkhMY(+s^ZP&F6K?mUJI+tVwiXV&}=e2ep_p|51ppmwKOV<}Lr{i44YPUXFQ5u~_gXC+^=>bXcxK$}l*u1=aw|qC0Vu z_|>3;=u#_1&Tn1L^co0KIq+L**YEFX)Z zvl5@WB5P_XLgC7Zi{&&P!(K2V=v&7wl^U(PE`CZv-^&+J8*^l2WWS=p!?qCZlk304 zlVXWiWtw0^(PI-qcN)pQ}H=~_pVM+Z- zC@^QXaoJj!E~Ji=S@b0W?TT=IqO8EpS6+VqpYkm(Ru}6YY+WO*9#EQS_*3TNU!@u2 zqL{O372kjaJZyxgkWzLM%~f9Mh=(k~X3I6Ht$GEF#GOaxs5_|?F~j>*j9y21qU%S+ zh!0&8+~R1XOK<#DnfU8_(IPxK>TRd}z?z$xSTXOGX$J5!ZWa2RV4oQ*B0I>dS1Ar`SAPaG17I6bXniz*|3vcjV(s$YfmuPG{DwCzV6)R-!n9?A{*X-i4A8R=68kjp4^_LSV*12*sQy z31NkI2sJ?~L-bOY78lvQ=dHKMHY=2@{fa?X;@%km86Pt4u!8iW6Ne3h znH8cvU{!T-Wi(@4IWMXR`((rxV!u3hm}_JAg6B|FV)tTEa`v8LpWcJ?agHHX=cU!Z zYS&Dj8~+O6;|c^H4;z1-PJje(dfM_n1USf7uRFq7^Tafs$|cN4@!z3TooG;hJIP2-n}Wz~qJ}jki>C&%YcK|ZDhtb-_%6-5(DB67H(ZNf z3+JV$DfLPF;CkMiCY?J`(WL65rMWeF@PI%^3SSZDfrh`bW1{S1I?$uidpFc~HxTW| zYAzB8ibcW1*u>!yzUMjAKYFxR#9%HO;9t6~yeP;ksLH)1e&=(vy25u}e)EmfZq$fw zRkDeT0*nI=m}_tolm02X3`1r&awYh+uCKuDdoU0E2YtZI;)$tChkjCfUWAQIQHIb| zI`?7t^;aei@T5ez5hsR%QO>fJ=juf_xsx?9eBs$iHf-TBv1`=id+CQgwp54D?aol$ z*;lJ@BsG{hb!DMDsvQeHJm0A%PNOz;F+%vG zbbT<7wE%(xx68NUdy-Mt0Zm$YP3F`g_SOY|*9Y#3#tM>U^wk?;ni<+m;Y`&SI z481JVLGU6NMwe0}Y3

FbmNl`Cpxs>Abf(7}J?3v-7VjC6ox~JeKd++BBjZ9;Kbj zADJ3J@C;64@os3KGzn$cL;cyEI87QdmM15*7Ia)Sni2YT-mY~+wZ;=rHFk!rUTI6;1LUpKBwnPS_0Z`o*e0PrKe_!&TWPm1R2LXz4+Y}BaX&nerTCtqs4czTBJ#8nhXE*?B}tGiP%?Q3B0{;Y zh1AHw$~NQG7x%hmr73%Lq}__D+Ge@hHsT7>Tqc@CMmBeNK=q!mV!pBU*uR0dl3#>J zQBlkx2S4VO!DZT!9uI=d-wZ)EVeZ&}&91rf+%8<}o%@e8w!jlxzhnlk3|k4P;#Gw8 z(<6l@5__=hgSThJ4+eG8XL$aou`VxadVNn@ zLrjILa3BzGDhcQ*IJk`QV{AlS(|tAr^zqI({%|8Y4j?H_`Jcg$`u?Nl-rISG2MF(-N{Zu4#w3gf zL5oK_!ML}948W`sK6~4ffqP{R@O$`9#o);JQR7(9-R_4@f&%2bvV;Z_BT)mRSNY_h zL#-Eh3%PB66dmT}p&=KBO%mSztTaz#G(bAtpUpS)z^@fpiWbW4JHk6rk@XSoY7A!?cT(mnx8M)&m5`!$R(X%6MTL1sF?KMmYr@EnVi6p>2EZSf zaDz!m{a(HTjnX0Mz?0B??eq>?$S)Dmw|U?%U?g|^YGdMiCAzKTP3aw8pI=C*oyk1y z96&HEL{43lTOy5N4(CkGnpm6$rbD=eoAhyf#kXEdY}~}t;)$78&sBL1i^7hq1N!vR z^^3f?Qq{$l>a|dqlV}mVTxxLHAZ_1<-J9ecdSL?OO=uK=s{Yy&0=5mTtPx~aP19^S zbKV6RtQg!JIq-#dyNO{@uQ%0V5R99o!~CLw(xR2I%RL{D?2zw)KfKPOM-z6EHwX-S zyDswEJeVI=?k5lZG%}hIe#4D(!F$e`HiaFPQ@sGc4JnFcP$JUCC$N(%^6GT`=~u2X z-w=p*RHvBju2jE76VD=kR!eXAUtW&|yb9*{ zIBopU&ys-c+q>R|m@j51(aZy2Ar_ghAFJpv^x$bBno~_A%z@P8qt{QN38uCFBv-QHt3p~mv3A&vZ#A0TNa=xlirZy{WA%mCvH*%+Lb^)*p37LtvKn>> z`@FnlG2{hlu&fMIM!Uf4KkCaG5n5yy09PXfzKlC6=Uks275agT z`DFlX5KXKbSzA7;mYSQbk6_Iyp=cYQLIR|Sh(sV+<>pFmFGxvji_;6l_*pkl>UO%h{(9DDdHhds}ub|LcCG@%a& zZC~MA&{e^dM>h zy%1>^x4`FMsRFoG_0^)JWa^94JGeGY%*(Z{?QN#sa}x57(#^`Thaz6Ty{SntlW%&3 z4A|%-k+CM8yt8m{Ne7I>+1fl?fa3rL*9IsOXtqcTiDPQvhvMdAw)+M~FmYTThgjk( z@nbikfPvkKLGdp0o|$>JI3t?qzmz~^Y^f1VB-&%gIPrDPl9Stf z_YaNdpqjo*w`1FHE31ZLz(Dk$!Ms#N7<%Io_F=}P065V)5QQPi2axX|+0}u)OY(wc zJ~PiaYMmWx)c7);JYeepz4N7)W9u?#sMGr3#$$a;zAKXHdRI%;Rg1X0iP7CP0XVP+`z;~oN|>N%G5a+lb%2sl?+1h!h6nF( zD&Rh`jBCNPNB}-8D4Y>Hm3hnRZqzBh|J5mhEc=dh5s4$-(3ZQ>SbpvCh&5#bYs6Jp zWl;1<_5xYCs=CIw%JJw}@DV#4YBc5N`c3~0-yUrg5Efvap#(4~wjLlTS7nVkn*nUX zOn^1N3Ar11hD9_D#O6pGHlB@-QRVJc_2ypaW?hK8-UgxS{ZC%}`1-V?PCoxaA~a1c zT0|nsomqZxgKzY}L``aGUckASL~Lq+6>G{jD4&MnVriX)WL1MUe8F_`j!OwMMbrXC z<45$`^Sac!uUMRwC==I#iN0(KT?kDjx}(8PqBj6(AfH>4&ZQCvWG&L#vHhis1r8ZP z=di$Rnod4@PKB!F9N5vYhXYIE-()2e(l5&S4}ehs7GV=Ki!xCm17Jacj}c6~sJidE zUy)+fYo$J9P6JE-NOPC||FZAj;VQEL9_v?%15A_lZ#_egX9RZTI5IryZb}xHyjDA~ zBHw*}#Md@z0ihswux6l%sxfV}QIqBt=rdp|1@km0r?^;3DuGp~KqswPD{&lKk?X&o zI;SkuC|%~U5Ui6YEIWljH#0}ytTFh{UP0hXqQC^+2dPfImKV7?pdJq@|ER=Pf{hQ) z(hmB93FCmxr+)&dchcj6F$Q9G?E>W>gQ<)jm^H08_bze@>(Yo6<504Cl$;hw=Sh|U zrBqnfI7!VjVQjhdE9KMqwl+)@B8()8R`N3)$6F{zng&p1Z8J9q@JlinSduehZ~>9y zA+Q8ppvodb8QTPi_&Ur55?zP|+Bv2R9fjAI1hZ{IHoUwr2&RH@>YOJxKcBZM`xwL9 zI|C6bB*kRo=coHUV8&96xL-+p#8@Z>0_j`;^y7nxYCKEO3#kzAiBJ9*8ZPmQEd_kl zp9T4EJptHxkVUj61p_hhTt`fu{hOS{0D2&;Asks0q+);x6NqwlN zAS_~jMzE^n@?yDq%p8;jEc81@;WrAJPJu^A??>sIOB_FrbEgWijedwkNg*KZG6_E+QC_8p0MjGvRFnZlBPI*jC4Wj6 zlU&)gVj|F^>^U+k&Y`Y8oGH1a>w2)=DhSN*aU4Jc1J`2Uf1*^gq-CR~`FN z^zsSjfL#A&?Eh+2w9V-rVF%@#po76gn8$~L$7&%`0NhGZ3&!i=1@UR?!t1zkwN+2Z zbJwnpXwsK1QB*zV$ecG$A&Nv%lRnVuN$U#-qea%rC44JDV9)ovf)>35H2UyL#TH;x zm;iTvm0XjF!Vgs^VVr%t1PN7Jx&nsm4=UBk2C|cp71TFopdc2?i{TE1_!9&^y&J{< z#_VFcC~hCeLB|!@KsKU475}ed-xe_*00qfin|r~}yzDf*zx2j0MFeWZxL5)`X-AdL z_gR3e0{aM9B80hP){R+&sZ&~ydInD;cWQ`_34A)uN#+u(iXj$C<(#G^*-x%r+roL&-{x>ij=?AFT!y->v{4(OQC_gcZLB#1=?bf!VlJUWSY>D-l&RU>{6WHP1O> zYmW+ug>kg$slIE8siN#lve1`g;{r{LJ0db9O-x$Yx5Ki4(zrkpQU(hnAj$BGY$R8c ztAtGz@z>QS2?196pOq)c(d_CxhwU6sH6_695#p0K5zrlXV?gU&QV;1Eyt7v z$zD_!Sl)kf&3#LQ&*x1BMS=%xgY&b&9clyeR|K zz(nz60TZl=1(qYCHa74d-|YK}`qR6L-)cN2_l1Dy577xU0G$#YkPiWK4Fj5bgV}4N zx$vNz>LApuNMq4}jwcl;3Gt5TwUo?=e%ZgWL$V4Nh&3{9Sxf?B(Lw~Iau`iH^8Gq- zA56vMi(XaUucY|ZJ?A1@=m$YF(52Kj(ib3*n;wln>zQ`LO9$wuyrRUil}MThtl})( zkZL6RZueFHMFYU4+ifwR%J+)UboZ8Q(ECrX{#Gz1u-3X9jamS1Jc%DyiJd`j_0(iq~^WA4!xBC;L~wQ*;0)TPG2@6ZdT#nP|<6&O&g z1%`mEW5j*;jqkvu*!-qOq|!OPwO^3`? z*K4ppC4PF(0-IZX@>MvUw6a6hHr~)hIrQ232)m&4C)jYvsoNHqnD#WTdeRI8BQUyJ zl%f%NqYJ0WJE?=y_&OtRZs*%VhEM@@?TOnXtyEDr&EoU<4F}#N+Z{(_DL8)uryWbx zfj64O_wU5L3JU*Z^4gh&h7s5Fw1bd_Z}XDm~B`5mwHmZ?R-E#jDS9o#Vfr~+fv}!lm70P_=q3Q%Ktj7183%7>kBS? z7+V9{vkw|;4od;KrE@!t31oHhd)Q08r0TI-DtRG%JJqRdcA*eUZRk3+4lKVLv4j`}PwRQ~k& z{I+^Abit((b_l

JUhcknkQsiPOMphxPKO6B6EB`J*n77qC-AoWE~f?|by~`3ulR zGI3)^58MTA1W&PTClC^(xBwP4F##uHg##MIWaR@#=>F<8THN(y>a*!#ov)sM35*9= zU`qX~1B6tOW6eQ)O^w9CAI_j2lqo+i_ni6bNT=TgAU9C1fss`W6?4L4ngHA~1{>U| zd$og(90k^?2hlr`)S+Vs?e^&7=XA^c!Jgzk_MTs6>Q5xILdpS7o*>#d14-P+O~Cnp zVe?n_R57PoF^@4s^Cph;(dH+qFFyNWiZ=i$A7J(gZ4&hCkK;fw;};Hl7Qgk&_^QPr z)~SUU1;&ES7{MX@1c(hTa?a??sTn2Iq+R|2f8@QLTf{!}q+JSR51_iuX07~|zDw+= z831$hZR>_;f3)!hSy{!al8 zWR8p_8XXQ@l+=&ovyg~lTp8wc10Fn_s}cQ+XRtw%#TZCQR-Ftx5XXRH5yA-u?F4EM z4m-u*HGwKYpG!e0Lr#)rnEX@{hWz+uD&Ie(-Esd|F!0e5Wagl0OpY_{YDF>}MMT_W zG*I#_v|(jy6-k|=~tOz!dNvks}2 z8B|@7N}S|T>vl72iNo=xFsWZnsB@XP*Ckw50SavHw_ThL_KPHb`YS-bTutJr&h&X? z4Dg%&NeE2I@pn`-w1(7@&`Qy9ddtZ2DtN>bK`z2rZDQObpHB#Q2Fh*UXoOPA^6Vf0 zT7A6I;7A4;)oXV@{dwzcf6iKtqz=FN7@!&xa}9J;fFTg64MrK&KE4%=>fbH-p?G1szAZQ5ZEx%c+bU1ty z!a5_lj4u)6eHq#b9~c2gqCkAbFW_v+9oW||?LxFj*!lpRBOXvHB?`{){GZ^W6+e^Q z@XCVH$=;LW<`h62{>NPtSRDJ{F?J8T4i1~PWD&}QKXR?g@?h35k+?OxNC#5}-5>H@ z3Wm6@>0C9jdV}oyN1cYz*P1Rr;mL}}=_z(PqRUyB1rlp1%L9?;WB8yi%vTk*%3QY;DA{whxG}efggfh)go~GOuHu@;HSx4{g<#M z3P7DYc^1*92W&$%ATsO(DSFP=CILXBI-NyaCa>K@J%vA>vAH3B)~wOBmz7UDjpNr9 z>av?fw^0)sd_)!53*F z5gexx8DB4O#E58zoAi6Nz6uLi;E2m$RpEbTA4rth49Hh0nd6l=eLGJJw$dI>T9*qEeQLhegGp0l~^fjP=E@0|Q zc0f^+s>0X!E)eI_;#79bv*q%B@rPa;M-V6txp&#XlRJ*$gtHwZ`8TI9fjik6_LD${ z1X0_*#&`i)Qd)K51}=V zsyZ4Rl|cMqj;J_`*MX?FNWS+dfLrH>4mFG|EPb977~wC!Nc+tU1z=J4)E zx+QBV9~yy|od?PmWLYJ@A=Fp6i;Fb@^$V*DNWz)T$IEm_*h!9o8wM5g$QL;Pp)dN~ zl=O$U_);@4L$)xV)^6oDNdku>|Buc_+uvga{o5i%!WZE{rj;1Ky)L5l3vBSQG)2bm zSJ3L@cRNPoU0Y9Dg|Uj(53V~YZxpNb>KF#ouc zq)~f$D~FKNU(MeGEGNWwQ~!zxtj`+YCL+J<9Z@5NuD>UwRoOZJT<{rdic8V8t` z$CaVAGwqS72F}Z@66j4&;!m3SHo^Exym)x1H}B#Ldk&GZL!#U4!x(W25kDC?HmQ{x ztACsPP3Y-?-%LJDT=UwuGq{*+=_EyWyRUMS`$~}eU+b+KaVzI)fs^V#HgpN0c}DP zf2&MyH~$W4N<)EUP`)(Ka$Gc3B0s_riXnHV-nFnWbTT~-L?Mn zU*EtBdVd_*Os#uV(!^cmZShp`%7)xMnw|7T^g0Vjk6*_-@OD7d7Z0cOK~^j=ld=&E zduw#iElcK5*E_G2%^f}TtcAP5k4wn2%eP=#Cz44i zB-bGBJLSQ#g7zD`hKDMyEMs+h67CL|bFyODuO^PTAwYKO5DlVvz*z)SaEyxmf|4cI z32LprpHXiSxOzV~Fg#zPn1xwy}+jc2s{{Id1PGx-M^C=M*RX3cq>v?Gb5qT{? zp7rFNqMvO#cyM4ie`rU#wz2toyh-V=&))uTsFw;0dBjMNPek#=&(T49suUlyUnJ+r zD;~c9NsYhvUutNV_j`8d4zmx#I`*Ur<*AFGH+7UI{FS28TX>K8tmOW%D6gS$Z5E_? zZ{6(Xw8C-kkoLfVGvDz0zfZYQ^capM^lP%$jY00C3DQJ_r(Pd$kLfOgdR#j~0NLp#X6#l=7;rqBp2K zjCkgKJ4Rh4cC$4~Dm}6wXxpQrsP8%1VdNtxNq=$v#GSuk1+#B(mTe`->5?r3*?hvf zq_)C$7zWt8yHmt`-pj0vU6nP_UGEO#dbHoC6?BJ&)LCMSCc;Hv%1*`YL7FWuhd7_s z+~8AF!n%hme>@8&rtq;CL}%_vl;Mj4X(bA4V3mt88{Dd- z1iIlh^gN<*wmjp}W9Z01dFwi6_Kc@?0vUJn_)fZs&A}A=uGVFh73fN25=dfHU?5*& z;d6U!KMNdo7K%TaHhI+qJvwYe#N`(nNBG#MpRZuBYYkT>-pw{|J>iZiEz?7A{-mF; z0j-AOtDo4RYa`&HtR}UK3(bFL9+^^oCgqXa(O*6kG-UZr{z_zxS@Gm0+#pZHr%b5TzOh?^2Wa zV-Y!(IYTLNXOb7@8Zlq@o?UOtzU)M!bnxwbS+~Qr$Ntzu_T|=3&jYYprV@TyqIYNeWzVJNSn2Lpd(UoEJX_4_UsruXtig zuf1FMf{|tLvpD8trk9-M`I39&Z0MP#?#_yL_I1gOtc>?534=M#pd}qmb?Dt)U4x16 z^I@8N;HEb1_PCOJ^W7Sg!Pl>rE<5wNRG2<`E$HsG7r*svuHBe&D%tR?o9&)cEq*Pl zb0GT*XXEo8uKaZFg6JBPcSxoPJaOFTddqGP+Z7I>U8C2}oPYXH*udxIKPR=#(o)%l q_=;6Q8`#JG<3Rty7h@_lQj&e~>led2!2f>$wZqx{hn#H(qW>Ro%`&6_ literal 0 HcmV?d00001 diff --git a/client/lucy-desktop/src/renderer/index.html b/client/lucy-desktop/src/renderer/index.html new file mode 100644 index 0000000..4dd41d5 --- /dev/null +++ b/client/lucy-desktop/src/renderer/index.html @@ -0,0 +1,13 @@ + + + + + + + Lucy + + +

+ + + diff --git a/client/lucy-desktop/src/renderer/src/components/AuraGlow.tsx b/client/lucy-desktop/src/renderer/src/components/AuraGlow.tsx new file mode 100644 index 0000000..f1ad2fb --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/components/AuraGlow.tsx @@ -0,0 +1,47 @@ +import { useEffect, useRef, type MutableRefObject } from "react" + +// Status-als-Licht: reaktive Aura hinter Lucy. Farbe = Status (hört zu / denkt / spricht), Intensität +// pulst mit dem Sprech-Pegel. Liest audioLevel per rAF (kein Re-Render pro Frame). +type LevelRef = MutableRefObject<{ current: number }> + +const COLORS: Record = { + warming: [120, 130, 150], idle: [125, 211, 252], listening: [56, 189, 248], + transcribing: [167, 139, 250], thinking: [251, 191, 36], speaking: [52, 211, 153], + error: [248, 113, 113], +} + +export function AuraGlow({ status, audioLevel }: { status: string; audioLevel: LevelRef }) { + const ref = useRef(null) + const statusRef = useRef(status); statusRef.current = status + const phase = useRef(0) + + useEffect(() => { + let raf = 0, last = performance.now() + const tick = (now: number) => { + const dt = Math.min(0.05, (now - last) / 1000); last = now + phase.current += dt + const el = ref.current + if (el) { + const s = statusRef.current + const [r, g, b] = COLORS[s] || COLORS.idle + const lvl = audioLevel.current?.current ?? 0 + // Grund-Puls je Status + Sprech-Reaktivität + const base = s === "thinking" || s === "transcribing" ? 0.45 + Math.sin(phase.current * 2.2) * 0.18 + : s === "listening" ? 0.55 + Math.sin(phase.current * 3) * 0.12 + : s === "speaking" ? 0.4 + Math.min(0.6, lvl * 1.8) + : s === "error" ? 0.6 + : 0.32 + Math.sin(phase.current * 1.1) * 0.06 // idle: ruhiges Atmen + const intensity = Math.max(0.15, Math.min(1, base)) + const scale = 1 + intensity * 0.18 + (s === "speaking" ? lvl * 0.25 : 0) + el.style.background = `radial-gradient(circle at 50% 42%, rgba(${r},${g},${b},${0.5 * intensity}) 0%, rgba(${r},${g},${b},${0.18 * intensity}) 32%, transparent 62%)` + el.style.transform = `scale(${scale})` + el.style.opacity = String(0.5 + intensity * 0.5) + } + raf = requestAnimationFrame(tick) + } + raf = requestAnimationFrame(tick) + return () => cancelAnimationFrame(raf) + }, [audioLevel]) + + return
+} diff --git a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx index e1893fb..229530f 100644 --- a/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx +++ b/client/lucy-desktop/src/renderer/src/components/Avatar3D.tsx @@ -2,7 +2,7 @@ import { Canvas, useFrame } from "@react-three/fiber" import { OrbitControls } from "@react-three/drei" import { useEffect, useRef, useState, type MutableRefObject } from "react" import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js" -import { Object3D, Vector3, AnimationMixer, LoopPingPong, LoopOnce, type AnimationAction } from "three" +import { Object3D, Vector3, AnimationMixer, LoopPingPong, LoopOnce, AdditiveBlending, DoubleSide, type AnimationAction } from "three" import { VRM, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" import { VRMAnimationLoaderPlugin, createVRMAnimationClip, type VRMAnimation } from "@pixiv/three-vrm-animation" import type { Emotion } from "../lib/voice/sentiment" @@ -281,25 +281,72 @@ function VrmModel({ url, audioLevel, emotion, status, cursor, pat, dance, onErro return vrm ? : null } +// Hologramm-Beiwerk: Projektor-Sockel (Glow-Ringe am Boden) + schwebender Kristall daneben. +// Reine Additiv-Meshes -> berühren die VRM-Materialien NICHT (voll reversibel per Toggle). +function HoloRig() { + const ringA = useRef(null), ringB = useRef(null), crystal = useRef(null) + useFrame((s, d) => { + const t = s.clock.elapsedTime + if (ringA.current) ringA.current.rotation.z += d * 0.3 + if (ringB.current) ringB.current.rotation.z -= d * 0.55 + if (crystal.current) { + crystal.current.rotation.y += d * 1.1 + crystal.current.rotation.x = Math.sin(t * 0.8) * 0.3 + crystal.current.position.y = 1.18 + Math.sin(t * 1.6) * 0.04 + } + }) + const cyan = "#4fd8ff" + return ( + + {/* Projektor-Sockel: weicher Glow-Disc + zwei rotierende Ringe am Boden (Fuesse bei y=0) */} + + + + + + {/* schwebender Kristall neben ihr (Schulterhoehe) */} + + + + + + ) +} + export function Avatar3D({ url, audioLevel, emotion, status = "idle", cursor, pat, dance, controls = true }: { url: string; audioLevel: LevelRef; emotion: EmotionRef; status?: string cursor?: CursorRef; pat?: NumRef; dance?: NumRef; controls?: boolean }) { const [err, setErr] = useState(null) + const [holo, setHolo] = useState(() => localStorage.getItem("lucy_holo") !== "0") // Hologramm-Look, Standard: an const statusRef = useRef(status); statusRef.current = status + const toggleHolo = () => setHolo((h) => { const n = !h; localStorage.setItem("lucy_holo", n ? "1" : "0"); return n }) return ( -
+
- - - + {holo ? ( + <> + + {/* Rim von hinten -> Silhouetten-Glow */} + + + + ) : ( + <> + + + + + )} {controls && ( )} + {holo && (<>
)} + {err && (
, , " + ", , oder . Er wird nicht angezeigt oder vorgelesen " + "und steuert nur deinen Gesichtsausdruck — wähle ihn passend zum Inhalt deiner Antwort." diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/sentiment.ts b/client/lucy-desktop/src/renderer/src/lib/voice/sentiment.ts new file mode 100644 index 0000000..10d781e --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/lib/voice/sentiment.ts @@ -0,0 +1,14 @@ +// Leichtgewichtige Stimmungs-Heuristik → treibt Lucys Mimik. Regelbasiert (kein Modell). +export type Emotion = "neutral" | "happy" | "angry" | "sad" | "surprised" | "relaxed" + +const RULES: [Emotion, RegExp][] = [ + ["happy", /(super|toll|klasse|freu|cool|prima|perfekt|danke|großartig|wunderbar|gerne|haha)/i], + ["surprised", /(wow|wirklich\?|krass|unglaublich|echt\?|tatsächlich|\?!|!\?|oha)/i], + ["angry", /(fehler|kaputt|mist|verdammt|nervt|schlecht|problem|ärgerlich|leider nicht|geht nicht)/i], + ["sad", /(leider|schade|traurig|tut mir leid|entschuldigung|sorry|bedauere)/i], +] + +export function sentimentToEmotion(text: string): Emotion { + for (const [emo, rx] of RULES) if (rx.test(text)) return emo + return "neutral" +} diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/usePushToTalk.ts b/client/lucy-desktop/src/renderer/src/lib/voice/usePushToTalk.ts new file mode 100644 index 0000000..6ced677 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/lib/voice/usePushToTalk.ts @@ -0,0 +1,45 @@ +import { useCallback, useEffect, useRef, useState } from "react" + +// Push-to-talk-Aufnahme via MediaRecorder. start beim Druecken, stop beim Loslassen -> Audio-Blob an onAudio. +export function usePushToTalk(onAudio: (blob: Blob) => void) { + const [recording, setRecording] = useState(false) + const recRef = useRef(null) + const chunksRef = useRef([]) + const streamRef = useRef(null) + + const start = useCallback(async () => { + if (recRef.current) return + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (e) { + console.error("Mikrofon-Zugriff verweigert:", e) + return + } + streamRef.current = stream + const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/webm" + const rec = new MediaRecorder(stream, { mimeType: mime }) + chunksRef.current = [] + rec.ondataavailable = (e) => { if (e.data.size) chunksRef.current.push(e.data) } + rec.onstop = () => { + const blob = new Blob(chunksRef.current, { type: mime }) + streamRef.current?.getTracks().forEach((t) => t.stop()) + streamRef.current = null + recRef.current = null + setRecording(false) + if (blob.size > 1200) onAudio(blob) + } + rec.start() + recRef.current = rec + setRecording(true) + }, [onAudio]) + + const stop = useCallback(() => { recRef.current?.stop() }, []) + + useEffect(() => () => { + recRef.current?.stop() + streamRef.current?.getTracks().forEach((t) => t.stop()) + }, []) + + return { recording, start, stop } +} diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts b/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts new file mode 100644 index 0000000..27fca5d --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/lib/voice/useVAD.ts @@ -0,0 +1,121 @@ +import { useEffect, useRef } from "react" + +// Freisprech-VAD (Voice Activity Detection): lauscht dauerhaft am Mikro, erkennt Sprech-Beginn/-Ende +// per Energie und liefert komplette Äußerungen als Blob. RÜCKKOPPLUNGS-SCHUTZ: `paused` (Lucy denkt/spricht) +// stoppt die Erkennung -> Lucys eigene Stimme triggert das Mikro nicht; + echoCancellation + Abkling-Sperre. + +interface VADOptions { + enabled: boolean // VAD-Modus an? + paused: boolean // Lucy beschäftigt (thinking/speaking/listening) -> nicht aufnehmen + onUtterance: (blob: Blob) => void + onListening?: (active: boolean) => void // UI: gerade Sprache am Aufnehmen? +} + +// Erkennung per ZEIT-DOMÄNEN-RMS mit ADAPTIVEM Rausch-Pegel (kalibriert sich aufs Mikro) — robuster +// als feste Frequenz-Schwellen. Schwellen werden relativ zum gemessenen Rauschen berechnet. +const START_FRAMES = 3 // so viele laute Frames in Folge -> Start (gegen Klick-Fehlstarts) +const SILENCE_MS = 900 // so lange Stille -> Äußerung zu Ende +const COOLDOWN_MS = 450 // nach Pause-Ende kurz taub (Echo/Lautsprecher abklingen lassen) + +export function useVAD({ enabled, paused, onUtterance, onListening }: VADOptions) { + const pausedRef = useRef(paused); pausedRef.current = paused + const onUtt = useRef(onUtterance); onUtt.current = onUtterance + const onLst = useRef(onListening); onLst.current = onListening + const resumeAt = useRef(0) + + // Abkling-Sperre: sobald Lucy fertig ist (paused true->false), kurz nicht lauschen + useEffect(() => { if (!paused) resumeAt.current = performance.now() + COOLDOWN_MS }, [paused]) + + useEffect(() => { + if (!enabled) return + let cancelled = false + let stream: MediaStream | null = null + let ctx: AudioContext | null = null + let raf = 0 + let rec: MediaRecorder | null = null + let chunks: Blob[] = [] + let speaking = false + let silenceStart = 0 + let voiceFrames = 0 + + const setListening = (v: boolean) => onLst.current?.(v) + + ;(async () => { + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }) + } catch (e) { + console.error("VAD: Mikrofon-Zugriff verweigert", e); return + } + if (cancelled) { stream.getTracks().forEach((t) => t.stop()); return } + + const Ctor = window.AudioContext || (window as any).webkitAudioContext + ctx = new Ctor() + try { await ctx.resume() } catch { /* */ } // WICHTIG: sonst bleibt der Context 'suspended' -> nur Nullen + const srcNode = ctx.createMediaStreamSource(stream) + const analyser = ctx.createAnalyser() + analyser.fftSize = 1024 + srcNode.connect(analyser) + const wave = new Uint8Array(analyser.fftSize) + let noiseFloor = 0.015 // adaptiver Rausch-Pegel (kalibriert sich auf das Mikro) + const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") ? "audio/webm;codecs=opus" : "audio/webm" + + const startRec = () => { + chunks = [] + rec = new MediaRecorder(stream!, { mimeType: mime }) + rec.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data) } + rec.onstop = () => { + const blob = new Blob(chunks, { type: mime }) + if (!cancelled && blob.size > 2400) onUtt.current(blob) // Mini-Blobs (Klicks) verwerfen + } + rec.start() + setListening(true) + } + const stopRec = (emit: boolean) => { + if (rec && rec.state !== "inactive") { + if (!emit) rec.onstop = null + rec.stop() + } + rec = null + setListening(false) + } + + const tick = () => { + if (cancelled) return + analyser.getByteTimeDomainData(wave) + let sq = 0 + for (let i = 0; i < wave.length; i++) { const v = (wave[i] - 128) / 128; sq += v * v } + const level = Math.sqrt(sq / wave.length) // RMS-Pegel 0..1 + const now = performance.now() + // Rausch-Pegel langsam an leise Phasen angleichen (nur wenn nicht geredet wird) + if (!speaking) noiseFloor += (Math.min(level, noiseFloor * 1.6 + 0.004) - noiseFloor) * 0.05 + const startTh = Math.max(0.02, noiseFloor * 2.8) // Sprache: deutlich über dem Rauschen + const stopTh = Math.max(0.012, noiseFloor * 1.7) // Stille: nahe am Rauschen + + if (pausedRef.current || now < resumeAt.current) { + if (speaking) { speaking = false; voiceFrames = 0; silenceStart = 0; stopRec(false) } // verwerfen + } else if (!speaking) { + if (level > startTh) { voiceFrames++; if (voiceFrames >= START_FRAMES) { speaking = true; silenceStart = 0; startRec() } } + else voiceFrames = 0 + } else { + if (level < stopTh) { + if (!silenceStart) silenceStart = now + else if (now - silenceStart > SILENCE_MS) { speaking = false; silenceStart = 0; stopRec(true) } + } else silenceStart = 0 + } + raf = requestAnimationFrame(tick) + } + tick() + })() + + return () => { + cancelled = true + cancelAnimationFrame(raf) + try { if (rec && rec.state !== "inactive") { rec.onstop = null; rec.stop() } } catch { /* */ } + stream?.getTracks().forEach((t) => t.stop()) + try { ctx?.close() } catch { /* */ } + onLst.current?.(false) + } + }, [enabled]) +} diff --git a/client/lucy-desktop/src/renderer/src/lib/voice/useVoiceAgent.ts b/client/lucy-desktop/src/renderer/src/lib/voice/useVoiceAgent.ts index 5a965de..66b2e96 100644 --- a/client/lucy-desktop/src/renderer/src/lib/voice/useVoiceAgent.ts +++ b/client/lucy-desktop/src/renderer/src/lib/voice/useVoiceAgent.ts @@ -3,7 +3,8 @@ import { usePushToTalk } from "./usePushToTalk" import { useVAD } from "./useVAD" import { AudioQueue } from "./audio" import { sentimentToEmotion, type Emotion } from "./sentiment" -import { BOX_URL, TTS_URL, SYSTEM_PROMPT } from "../../config" +import { BOX_URL, SYSTEM_PROMPT } from "../../config" +import { stt, tts, SpeechScheduler } from "../../voice-core" // Voll-Duplex-Schleife: PTT -> Box /stt -> Box /chat (SSE Hermes) -> CHUNKS -> lokal /tts -> AudioQueue. // Hirn+STT = Box (wie WebUI), Stimme = lokal (pocket-tts, CPU). Antwort wird in groessere Bloecke @@ -13,6 +14,13 @@ import { BOX_URL, TTS_URL, SYSTEM_PROMPT } from "../../config" export type VoiceStatus = "warming" | "idle" | "listening" | "transcribing" | "thinking" | "speaking" | "error" export interface ChatMsg { role: "user" | "assistant"; text: string } +// Perf-Diagnose (Client): Zeit pro Stufe ab „Mikro losgelassen". Aus via localStorage lucy_perf=0. +// Perf-Diagnose: NUR Console (kein POST an den pocket_server mehr — der generiert Audio; POSTs +// mittendrin verursachten hakelige Sprachausgabe). Diagnose ist durch (Thinking war die Ursache). +const PERF = typeof localStorage !== "undefined" && localStorage.getItem("lucy_perf") === "1" // opt-in statt default +function plogSend(msg: string) { if (PERF) console.log(`[lucy-perf] ${msg}`) } +function plog(label: string, ms: number) { plogSend(`${label} = ${Math.round(ms)}ms`) } + // Hermes schreibt manchmal ASCII-Umlaute (ue/ae/oe/ss) statt ä/ö/ü/ß -> pocket-tts liest die falsch vor. // Gezielt häufige UMLAUT-Stämme zurückwandeln. KONSERVATIV: nur Muster, bei denen ASCII fast immer ein // Umlaut ist (echte "ue"-Wörter wie aktuell/neue/Quelle bleiben unangetastet — die sind hier NICHT gelistet). @@ -51,10 +59,23 @@ export function restoreUmlauts(s: string): string { const EMO_TAG = //i function stripEmoTag(s: string): string { return s.replace(//gi, "").trimEnd() } +// Hermes hängt bei selbst erzeugten Medien (z.B. eigener Screenshot via pc-control) einen Roh-Token +// „MEDIA:" an. Das ist weder Sprech- noch Anzeigetext -> überall entfernen (bis Medien echt +// gerendert werden). Betrifft auch Lucys eigene Bildschirm-Sicht, die den Screenshot ohnehin schon liefert. +function stripMedia(s: string): string { + return s.replace(/\bMEDIA:\S+/gi, "").replace(/[ \t]{2,}/g, " ").replace(/[ \t]+\n/g, "\n").trimEnd() +} + function cleanForTTS(s: string): string { return restoreUmlauts(s) .replace(/```[\s\S]*?```/g, " ").replace(/`([^`]*)`/g, "$1") .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") + .replace(/\bMEDIA:\S+/gi, " ") + // Zeitzonen-Codes (UTC/CET/GMT…) + evtl. angehängte Zeit: Pocket mangelt die Akronyme zu + // Kauderwelsch -> aus der Stimme entfernen (Anzeige behält sie). Lucy sagt die lokale Zeit ohnehin. + .replace(/\b(?:UTC|GMT|CET|CEST|MEZ|MESZ|PST|PDT|EST|EDT)\b\s*[+\-]?\d{0,2}(?::\d{2})?/gi, " ") + // Zahlen/Uhrzeiten (18:01 -> „achtzehn Uhr eins", 2026 -> „zweitausend…") normalisiert jetzt + // der lokale pocket_server (text_norm.py, num2words) — kontextsicher (Modellnummern bleiben). .replace(/https?:\/\/\S+/gi, " ").replace(/www\.\S+/gi, " ").replace(/\b\S+@\S+\.\S+\b/g, " ") .replace(/[*_#>~|`]+/g, " ").replace(/^\s*[-•·]\s+/gm, " ") .replace(/\s*&\s*/g, " und ") @@ -72,26 +93,6 @@ function newSessionId(): string { return "lucy-" + Math.random().toString(36).slice(2) + Date.now().toString(36) } -async function ttsToBuffer(text: string): Promise { - const r = await fetch(`${TTS_URL}/tts`, { - method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), - }) - if (!r.ok) throw new Error(`TTS ${r.status}`) - return r.arrayBuffer() -} - -// Streamt die Stimme (PCM16) und spielt sie lueckenlos ab, sobald die ersten Chunks da sind -// (Time-to-first-audio ~1s statt der ganzen Generierung). Liefert true, wenn etwas gesprochen wurde. -async function ttsStreamPlay(queue: AudioQueue, text: string): Promise { - const r = await fetch(`${TTS_URL}/tts/stream`, { - method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), - }) - if (!r.ok || !r.body) throw new Error(`TTS ${r.status}`) - const sr = Number(r.headers.get("X-Sample-Rate") || "24000") - await queue.playPcmStream(r.body, sr) - return true -} - // Bildschirm-Sicht: erkennt im Gesagten die Bitte, auf den Schirm zu schauen const VISION_RX = /\b(schau|sieh|siehst|guck|guckst|zeig|bildschirm|screen|monitor|fenster|erkennst?|lies (mir|das)|was (steht|ist) (da|hier|auf)|auf meinem (bildschirm|schirm|screen))\b/i function extractWindowName(text: string): string | null { @@ -116,6 +117,10 @@ export function useVoiceAgent() { const audioLevel = useRef({ current: 0 }) const emotion = useRef("neutral") const queueRef = useRef(null) + const schedulerRef = useRef(null) + const turnAbortRef = useRef(null) // laufenden Turn abbrechen (Barge-in) + const thinkingRef = useRef(false) // Hermes werkelt noch (Tools) -> Status nach Filler zurück auf 'thinking' + const turnT0 = useRef(0) // Perf: Startzeit des aktuellen Turns (Mikro losgelassen) const sessionId = useRef(newSessionId()) // Live-Status fuer die Eingabe-Sperre (ohne pressStart-Closure neu zu binden) const statusRef = useRef("warming") @@ -124,9 +129,15 @@ export function useVoiceAgent() { const ensureQueue = useCallback(() => { if (!queueRef.current) { const q = new AudioQueue() - q.onSpeaking = (sp) => setStatus((s) => (sp ? "speaking" : s === "speaking" ? "idle" : s)) queueRef.current = q audioLevel.current = q.level + // Satz-Pipelining: der Scheduler spricht einzelne Sätze, sobald sie aus dem Hirn-Stream + // fertig sind. Er (nicht die Queue) treibt den „speaking"-Status, damit es zwischen Sätzen + // nicht flackert (die Queue pausiert die Pegelmessung je Satz). + const sch = new SpeechScheduler(tts, q) + // Nach dem Filler-Satz zurück auf 'thinking', solange Hermes noch werkelt (sonst flackert's auf idle). + sch.onBusy = (busy) => setStatus((s) => (busy ? "speaking" : thinkingRef.current ? "thinking" : s === "speaking" ? "idle" : s)) + schedulerRef.current = sch } return queueRef.current }, []) @@ -137,16 +148,10 @@ export function useVoiceAgent() { useEffect(() => { let cancelled = false ;(async () => { - for (let i = 0; i < 120 && !cancelled; i++) { - try { - const h = await (await fetch(`${TTS_URL}/health`)).json() - if (h.status === "ok") break - } catch { /* Dienst startet evtl. noch (Spawn + Modell-Load) */ } - await new Promise((r) => setTimeout(r, 1000)) - } + await tts.waitReady(120_000) if (cancelled) return // ein Aufwaerm-Satz (primt alle lazy Pfade); Audio verwerfen - try { await ttsToBuffer("Alles bereit, Commander.") } catch { /* */ } + try { await tts.synthesize("Alles bereit, Commander.") } catch { /* */ } if (cancelled) return setReady(true); setStatus("idle") })() @@ -156,21 +161,60 @@ export function useVoiceAgent() { // Ein kompletter Turn: User-Text (+ optional Bildschirm-Bilder, Multi-Monitor) -> Hermes (SSE) -> Stimme. const runTurn = useCallback(async (userText: string, images?: string[]) => { const queue = ensureQueue() + const scheduler = schedulerRef.current! + // Vorherigen Turn (falls noch am Streamen) hart abbrechen -> kein „Weiterreden" nach Barge-in. + turnAbortRef.current?.abort() + const ac = new AbortController() + turnAbortRef.current = ac + scheduler.clear(); queue.clear() // frischer Turn: evtl. Reste aus vorherigem Sprechen verwerfen setStatus("thinking") + thinkingRef.current = true // Hermes werkelt (evtl. Tools) -> Status nach Filler zurück auf 'thinking' let assistant = "" + let firstToken = true setMessages((m) => [...m, { role: "assistant", text: "" }]) - let spokeAny = false, ttsFailed = false + // GANZE Antwort in EINEM Stream an Pocket (nach Hermes-Ende). Pocket kürzt intern den ersten + // Chunk (FAST_FIRST) -> schnelles erstes Audio, konsistente Prosodie, wenige Collapse-Regens. + // (Client-seitiges Satz-Chunking kämpfte gegen genau diese Optimierung -> verworfen.) + let dispatchedAny = false + let aborted = false + // --- Inkrementelles Sprechen bei TOOL-Turns ------------------------------------------------ + // Ohne Tools bleibt alles Ein-Stück (Flush erst am Ende) -> gleiche Prosodie/FAST_FIRST wie bisher. + // Sobald Hermes ein Tool anstößt (hermes.*-Event), sprechen wir die bis dahin FERTIGEN Sätze schon + // -> Lucy redet, WÄHREND das Tool läuft, statt am Ende alles am Stück (sonst 30s+ Totstille bei Tool-Ketten). + let spokenLen = 0 + const SENT_END = /[.!?…](?=[\s"“”„)\]]|$)/g + const flushSpeakable = (force: boolean) => { + const raw = assistant.slice(spokenLen) + if (!raw.trim()) return + if (!force && ((raw.match(/```/g)?.length || 0) % 2) === 1) return // offener Code-Zaun -> warten + let upto = raw.length + if (!force) { + let last = -1, m: RegExpExecArray | null + SENT_END.lastIndex = 0 + while ((m = SENT_END.exec(raw))) last = m.index + 1 + if (last < 0) return // noch kein ganzer Satz fertig -> warten + upto = last + } + spokenLen += upto + const seg = cleanForTTS(stripEmoTag(raw.slice(0, upto))) + if (seg) { scheduler.push(seg); dispatchedAny = true } + } + // Sofort-Kontext: lokale Zeit/Datum mitgeben -> Lucy braucht dafür KEIN Tool (spart den 13s-Tool-Tanz). + const timeCtx = `\n\n[Sofort-Kontext, DIREKT nutzbar OHNE Tool: Lokale Zeit/Datum beim Commander ist ` + + `${new Date().toLocaleString("de-DE", { weekday: "long", day: "numeric", month: "long", year: "numeric", hour: "2-digit", minute: "2-digit" })} Uhr.]` try { const r = await fetch(`${BOX_URL}/api/voice/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ text: userText, session_id: sessionId.current, system: SYSTEM_PROMPT, images: images || [] }), + body: JSON.stringify({ text: userText, session_id: sessionId.current, system: SYSTEM_PROMPT + timeCtx, images: images || [] }), + signal: ac.signal, }) if (!r.ok || !r.body) throw new Error(`Agent ${r.status}`) const reader = r.body.getReader() const dec = new TextDecoder() let sse = "" for (;;) { + if (ac.signal.aborted) break const { done, value } = await reader.read() if (done) break sse += dec.decode(value, { stream: true }) @@ -181,7 +225,16 @@ export function useVoiceAgent() { // Chat-Chunks -> nicht als solche parsen (ein Tool-Hickup mit error-Objekt darf die Antwort // nicht abbrechen). Optional koennte man hier Tool-Status anzeigen. const evType = lines.find((l) => l.startsWith("event:"))?.slice(6).trim() - if (evType && evType.startsWith("hermes.")) continue // Tool-Fortschritt (Status bleibt 'thinking') + if (evType && evType.startsWith("hermes.")) { + // Tool-Fortschritt (Status bleibt 'thinking'). Für die „denkt-lange"-Diagnose zeigen wir + // die Agent-Aktivität mit Zeitstempel -> so sieht man, ob Tools/Reasoning die Zeit fressen. + if (PERF) { + const dt = lines.find((l) => l.startsWith("data:"))?.slice(5).trim().slice(0, 140) || "" + plogSend(`Hermes ${evType} +${Math.round(performance.now() - turnT0.current)}ms ${dt}`) + } + flushSpeakable(false) // Tool läuft an -> die bis hier fertigen Sätze schon sprechen (redet WÄHREND das Tool arbeitet) + continue + } const line = lines.find((l) => l.startsWith("data:")) if (!line) continue const data = line.slice(5).trim() @@ -196,33 +249,39 @@ export function useVoiceAgent() { } const delta = json.choices?.[0]?.delta?.content || "" if (!delta) continue + if (firstToken) { firstToken = false; plog("Chat-TTFB (1. Hermes-Token)", performance.now() - turnT0.current) } assistant += delta + // Defensiv: falls Hermes seinen internen Umschlag durchreicht (Mid-Turn) -> NICHTS vorlesen. + if (assistant.includes("OUT-OF-BAND USER MESSAGE")) { aborted = true; scheduler.clear(); queue.clear(); break } emotion.current = sentimentToEmotion(assistant) - setMessages((m) => { const c = m.slice(); c[c.length - 1] = { role: "assistant", text: stripEmoTag(restoreUmlauts(assistant)) }; return c }) + setMessages((m) => { const c = m.slice(); c[c.length - 1] = { role: "assistant", text: stripEmoTag(stripMedia(restoreUmlauts(assistant))) }; return c }) } + if (aborted) break } - if (!assistant.trim()) { setStatus("idle"); return } - // Defensiv: falls Hermes mal seinen internen Umschlag durchreicht (Mid-Turn), NICHT vorlesen. - if (assistant.includes("OUT-OF-BAND USER MESSAGE")) { + if (ac.signal.aborted) return // Turn wurde unterbrochen (Barge-in) -> still beenden + if (aborted) { setStatus("error") setError("Nachricht kam mitten im Turn an — bitte warten, bis Lucy fertig ist, dann erneut fragen.") return } + if (!assistant.trim()) { setStatus("idle"); return } // Stimmungs-Tag (falls vorhanden) -> Avatar-Mimik nach BEDEUTUNG (sonst bleibt Sentiment-Heuristik). const emo = assistant.match(EMO_TAG) if (emo) emotion.current = emo[1].toLowerCase() as Emotion - // Ganze Antwort STREAMEN -> erstes Audio nach ~1s, lueckenlos (pocket-tts splittet intern in - // Saetze mit natuerlicher Prosodie). Kein Chunking noetig (real-time RTF ~0.74). - const spoken = cleanForTTS(stripEmoTag(assistant)) - if (spoken) { - try { await ttsStreamPlay(queue, spoken); spokeAny = true } - catch (e) { ttsFailed = true; console.error("TTS-Fehler:", e) } - } - if (!spokeAny) { + // Rest sprechen: bei Tool-Turns wurde schon segmentweise geflusht; hier kommt das letzte Segment + // (bzw. bei Turns OHNE Tools die GANZE Antwort in einem Rutsch -> FAST_FIRST, gleiche Prosodie wie bisher). + thinkingRef.current = false // Hermes fertig -> nach der Antwort darf der Status auf idle (kein Rückfall auf 'thinking') + plog("Text an TTS (Rest)", performance.now() - turnT0.current) + flushSpeakable(true) + await scheduler.idle() // warten, bis Lucy fertig gesprochen hat + if (dispatchedAny && !scheduler.playedAny) { setStatus("error") - setError(ttsFailed ? "Sprachausgabe fehlgeschlagen — laeuft der lokale TTS-Dienst?" : "Keine Sprachausgabe erzeugt.") + setError("Sprachausgabe fehlgeschlagen — laeuft der lokale TTS-Dienst?") + } else if (!dispatchedAny) { + setStatus("idle") } } catch (e: any) { + if (e?.name === "AbortError" || ac.signal.aborted) return // absichtlich abgebrochen (Barge-in) setStatus("error"); setError(`Agent-Antwort fehlgeschlagen: ${e.message}`) } }, [ensureQueue]) @@ -244,15 +303,16 @@ export function useVoiceAgent() { const handleAudio = useCallback(async (blob: Blob) => { setError(null) + turnT0.current = performance.now() // Perf: Startpunkt = Mikro losgelassen + turnAbortRef.current?.abort() // Barge-in: laufenden Hirn-Turn stoppen (kein Nachschieben) + thinkingRef.current = false ensureQueue().clear() + schedulerRef.current?.clear() // Barge-in: wartende Sätze mit verwerfen setStatus("transcribing") let userText = "" try { - const fd = new FormData() - fd.append("audio", blob, "rec.webm") - const r = await fetch(`${BOX_URL}/api/voice/stt`, { method: "POST", body: fd }) - if (!r.ok) throw new Error(`STT ${r.status}`) - userText = (await r.json()).text?.trim() || "" + userText = await stt.transcribe(blob) + plog("STT (Mikro->Text)", performance.now() - turnT0.current) } catch (e: any) { setStatus("error"); setError(`Spracherkennung fehlgeschlagen: ${e.message}`); return } @@ -266,7 +326,7 @@ export function useVoiceAgent() { // Auge-Button: Lucy aktiv auf den Bildschirm schauen lassen (ohne Sprachbefehl) const lookAtScreen = useCallback(async () => { if (!ready || statusRef.current === "thinking" || statusRef.current === "transcribing") return - setError(null); ensureQueue().clear() + setError(null); turnT0.current = performance.now(); ensureQueue().clear(); schedulerRef.current?.clear() let images: string[] = [] try { images = ((await window.lucy?.captureScreen()) || []).filter(Boolean) } catch { /* */ } const prompt = "Schau auf meinen Bildschirm und sag mir kurz, was du darauf siehst." @@ -300,7 +360,10 @@ export function useVoiceAgent() { const pressEnd = useCallback(() => { stop() }, [stop]) const reset = useCallback(() => { + turnAbortRef.current?.abort() + thinkingRef.current = false queueRef.current?.clear() + schedulerRef.current?.clear() setMessages([]); setError(null); setStatus("idle") sessionId.current = newSessionId() // frischer Gesprächsfaden (Mem0-Gedächtnis bleibt) }, []) diff --git a/client/lucy-desktop/src/renderer/src/main.tsx b/client/lucy-desktop/src/renderer/src/main.tsx new file mode 100644 index 0000000..f7629cd --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/main.tsx @@ -0,0 +1,10 @@ +import React from "react" +import ReactDOM from "react-dom/client" +import App from "./App" +import "./styles.css" + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + , +) diff --git a/client/lucy-desktop/src/renderer/src/styles.css b/client/lucy-desktop/src/renderer/src/styles.css index 6009ed4..850de3b 100644 --- a/client/lucy-desktop/src/renderer/src/styles.css +++ b/client/lucy-desktop/src/renderer/src/styles.css @@ -254,3 +254,29 @@ body { } .msg.assistant .bubble:hover .msgCopy { opacity: 1; } .msgCopy:hover { background: rgba(255,255,255,0.14); color: #e5e7eb; } + +/* ---- Hologramm-Look (Holo-Lucy): Silhouetten-Glow + Scanlines + Cyan-Tint ---- */ +/* drop-shadow folgt der Alpha-Silhouette des transparenten WebGL-Canvas -> billiger „Bloom"-Halo */ +.avatarCanvas.holo canvas { + filter: drop-shadow(0 0 5px rgba(90,222,255,0.85)) drop-shadow(0 0 15px rgba(56,190,255,0.5)); +} +.holoTint { + position: absolute; inset: 0; z-index: 2; pointer-events: none; mix-blend-mode: screen; + background: radial-gradient(ellipse 60% 55% at 50% 42%, rgba(60,200,255,0.12), rgba(30,120,255,0.05) 62%, transparent 82%); +} +.holoScan { + position: absolute; inset: 0; z-index: 3; pointer-events: none; mix-blend-mode: screen; + background: repeating-linear-gradient(0deg, rgba(120,230,255,0.07) 0px, rgba(120,230,255,0.07) 1px, transparent 2px, transparent 4px); + animation: holoFlicker 4s infinite; +} +@keyframes holoFlicker { + 0%, 100% { opacity: 0.5; } 48% { opacity: 0.46; } 50% { opacity: 0.72; } 52% { opacity: 0.45; } + 70% { opacity: 0.55; } 71% { opacity: 0.4; } 72% { opacity: 0.55; } 92% { opacity: 0.6; } +} +.holoToggle { + position: absolute; top: 6px; right: 6px; z-index: 6; -webkit-app-region: no-drag; + width: 26px; height: 24px; border-radius: 7px; cursor: pointer; font-size: 13px; + border: 1px solid rgba(125,211,252,0.35); background: rgba(12,12,20,0.5); color: #7dd3fc; +} +.holoToggle:hover { background: rgba(125,211,252,0.2); } +.holoToggle.on { background: rgba(125,211,252,0.28); color: #bae6fd; } diff --git a/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts b/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts new file mode 100644 index 0000000..e117fea --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/voice-core/engines/boxStt.ts @@ -0,0 +1,16 @@ +// Box-STT-Adapter — Spracherkennung läuft auf der Box (MC2 /api/voice/stt -> faster-whisper). +import { BOX_URL } from "../../config" +import type { SttEngine } from "../types" + +export class BoxSttEngine implements SttEngine { + readonly id = "box-whisper" + constructor(private readonly baseUrl: string = BOX_URL) {} + + async transcribe(audio: Blob): Promise { + const fd = new FormData() + fd.append("audio", audio, "rec.webm") + const r = await fetch(`${this.baseUrl}/api/voice/stt`, { method: "POST", body: fd }) + if (!r.ok) throw new Error(`STT ${r.status}`) + return (await r.json()).text?.trim() || "" + } +} diff --git a/client/lucy-desktop/src/renderer/src/voice-core/engines/pocketTts.ts b/client/lucy-desktop/src/renderer/src/voice-core/engines/pocketTts.ts new file mode 100644 index 0000000..9a13c54 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/voice-core/engines/pocketTts.ts @@ -0,0 +1,39 @@ +// Pocket-TTS-Adapter — kapselt den lokalen pocket_server (CPU, vom Main-Prozess gespawnt). +// Endpunkte: POST /tts {text}->WAV, POST /tts/stream {text}->PCM16 (+ X-Sample-Rate), GET /health. +import { TTS_URL } from "../../config" +import type { TtsEngine, TtsStreamHandle } from "../types" + +export class PocketTtsEngine implements TtsEngine { + readonly id = "pocket" + constructor(private readonly baseUrl: string = TTS_URL) {} + + async synthesize(text: string): Promise { + const r = await fetch(`${this.baseUrl}/tts`, { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), + }) + if (!r.ok) throw new Error(`TTS ${r.status}`) + return r.arrayBuffer() + } + + async synthesizeStream(text: string): Promise { + const r = await fetch(`${this.baseUrl}/tts/stream`, { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text }), + }) + if (!r.ok || !r.body) throw new Error(`TTS ${r.status}`) + const sampleRate = Number(r.headers.get("X-Sample-Rate") || "24000") + return { stream: r.body, sampleRate } + } + + // pocket_server serviert /health erst, wenn das Modell fertig geladen ist (FastAPI-lifespan). + async waitReady(timeoutMs = 120_000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + const h = await (await fetch(`${this.baseUrl}/health`)).json() + if (h.status === "ok") return true + } catch { /* Dienst startet evtl. noch (Spawn + Modell-Load) */ } + await new Promise((r) => setTimeout(r, 1000)) + } + return false + } +} diff --git a/client/lucy-desktop/src/renderer/src/voice-core/index.ts b/client/lucy-desktop/src/renderer/src/voice-core/index.ts new file mode 100644 index 0000000..67723c6 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/voice-core/index.ts @@ -0,0 +1,14 @@ +// voice-core — öffentliche API + aktive Engine-Auswahl. +// Aktuell fest: Pocket-TTS (lokal, Kyutai — macht auch Lucys Voice-Cloning) + Box-Whisper-STT. +// TTS-Engines beschränkt auf pocket (live) + f5 (GPU-Option). ElevenLabs/Edge/Chatterbox/Piper +// waren nur im alten Web-Pfad und sind hier bewusst NICHT dabei. Der Voice-Agent importiert NUR von hier. +import { PocketTtsEngine } from "./engines/pocketTts" +import { BoxSttEngine } from "./engines/boxStt" +import type { SttEngine, TtsEngine } from "./types" + +export const tts: TtsEngine = new PocketTtsEngine() +export const stt: SttEngine = new BoxSttEngine() + +export { SpeechScheduler } from "./speech" +export type { AudioSink } from "./speech" +export type { SttEngine, TtsEngine, TtsStreamHandle } from "./types" diff --git a/client/lucy-desktop/src/renderer/src/voice-core/speech.ts b/client/lucy-desktop/src/renderer/src/voice-core/speech.ts new file mode 100644 index 0000000..1742264 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/voice-core/speech.ts @@ -0,0 +1,84 @@ +// SpeechScheduler — Satz-Pipelining fürs Voll-Duplex-Gefühl (Etappe 2, Schritt 1). +// Nimmt einzelne SÄTZE entgegen, während das Hirn noch streamt, und spricht sie lückenlos +// nacheinander: erstes Audio nach dem ERSTEN Satz statt nach der ganzen Antwort. +// Barge-in-fest über eine Generation (clear() verwirft laufende + wartende Sätze). +import type { TtsEngine, TtsStreamHandle } from "./types" + +// Perf-Diagnose: misst hörbare Stille + Synthese-Zeit je Satz. Abschalten via localStorage lucy_perf=0. +const PERF = typeof localStorage !== "undefined" && localStorage.getItem("lucy_perf") !== "0" + +/** Minimaler Audio-Ausgang (AudioQueue erfüllt diese Form) — hält voice-core von der UI entkoppelt. */ +export interface AudioSink { + playPcmStream(stream: ReadableStream, sampleRate: number): Promise +} + +export class SpeechScheduler { + private pending: string[] = [] + private running = false + private gen = 0 + private lastPlayEnd = 0 // Zeitstempel Ende des vorigen Satzes (für die „Stille davor"-Messung) + /** true, sobald mindestens ein Satz seit dem letzten clear() gesprochen wurde (für Fehlererkennung). */ + playedAny = false + /** Aktiv-Signal: true beim ersten Satz, false wenn alles gesprochen ist (treibt den „speaking"-Status). */ + onBusy?: (busy: boolean) => void + + constructor(private readonly tts: TtsEngine, private readonly sink: AudioSink) {} + + /** Einen fertigen Satz einreihen (leere werden ignoriert) und ggf. den Pump starten. */ + push(text: string): void { + const t = text.trim() + if (!t) return + this.pending.push(t) + if (!this.running) void this.pump() + } + + /** Barge-in / frischer Turn: laufende + wartende Sätze verwerfen. Der Audio-Sink wird separat + * gestoppt (queue.clear()). Setzt die Fehler-/Erfolgs-Erkennung zurück. */ + clear(): void { + this.gen++ + this.pending = [] + this.playedAny = false + this.lastPlayEnd = 0 + } + + /** Wartet, bis alle eingereihten Sätze gesprochen sind. */ + async idle(): Promise { + while (this.running || this.pending.length) await new Promise((r) => setTimeout(r, 40)) + } + + private synth(text: string): Promise { + return this.tts.synthesizeStream(text).catch((e) => { console.error("TTS-Fehler:", e); return null }) + } + + private async pump(): Promise { + if (this.running) return + this.running = true + this.onBusy?.(true) + try { + // SERIELL, KEIN Prefetch: der pocket_server ist Single-Instanz. synthesizeStream liefert den + // Handle schon bei TTFB (Server generiert den Body noch) -> ein vorab angestoßenes Segment + // würde den laufenden Lauf KONKURRIEREN lassen (gemessen bei Tool-Turns: gen 17s statt 5s, + // TTFB-Spitze 14.7s). Gaplessness INNERHALB einer Antwort macht ohnehin der Server (Satz-Split + // + _compress_gaps); hier werden nur die Segmente/Tool-Häppchen der Reihe nach gesprochen. + let idx = 0 + while (this.pending.length) { + const myGen = this.gen + const tWait = performance.now() + const handle = await this.synth(this.pending.shift()!) + const synthWait = performance.now() - tWait + if (myGen !== this.gen) { if (handle) handle.stream.cancel().catch(() => {}); continue } // ge-cleared -> Server sofort freigeben + if (!handle) continue // Synthese fehlgeschlagen -> nächsten Satz versuchen + const gap = this.lastPlayEnd ? tWait - this.lastPlayEnd : 0 // hörbare Stille seit dem letzten Satz + const tPlay = performance.now() + await this.sink.playPcmStream(handle.stream, handle.sampleRate) + this.lastPlayEnd = performance.now() + this.playedAny = true + if (PERF) console.log(`[lucy-perf] TTS Satz #${idx}: Stille davor=${Math.round(gap)}ms · Synth-Warten=${Math.round(synthWait)}ms · Spieldauer=${Math.round(this.lastPlayEnd - tPlay)}ms`) + idx++ + } + } finally { + this.running = false + this.onBusy?.(false) + } + } +} diff --git a/client/lucy-desktop/src/renderer/src/voice-core/types.ts b/client/lucy-desktop/src/renderer/src/voice-core/types.ts new file mode 100644 index 0000000..d465768 --- /dev/null +++ b/client/lucy-desktop/src/renderer/src/voice-core/types.ts @@ -0,0 +1,27 @@ +// voice-core — Engine-Adapter-Interface (Etappe 0 des Greenfield-Umbaus). +// Ziel: STT/TTS-Backends hinter EINEM Vertrag, damit pocket | f5 | piper | elevenlabs | edge +// austauschbar sind (Config/Picker) und der Voice-Agent nicht mehr direkt an URLs klebt. + +export interface SttEngine { + /** Stabiler Bezeichner der Engine (z. B. "box-whisper"). */ + readonly id: string + /** Audio (webm/opus/wav …) -> erkannter Text (leer, wenn nichts verstanden). */ + transcribe(audio: Blob): Promise +} + +/** Ergebnis einer Streaming-Synthese: fortlaufender PCM16-mono-Stream + Samplerate. */ +export interface TtsStreamHandle { + readonly stream: ReadableStream + readonly sampleRate: number +} + +export interface TtsEngine { + /** Stabiler Bezeichner der Engine (z. B. "pocket"). */ + readonly id: string + /** Vollständige Synthese in einen dekodierbaren Audio-Buffer (z. B. WAV) — für Warm-up. */ + synthesize(text: string): Promise + /** Streaming-PCM16-Synthese für niedrige Time-to-first-audio (die Live-Antwort). */ + synthesizeStream(text: string): Promise + /** Bereitschafts-/Warm-Gate: true, sobald die Engine antwortet (Modell evtl. lazy geladen). */ + waitReady(timeoutMs?: number): Promise +} diff --git a/client/lucy-desktop/start-lucy.cmd b/client/lucy-desktop/start-lucy.cmd new file mode 100644 index 0000000..54b36f3 --- /dev/null +++ b/client/lucy-desktop/start-lucy.cmd @@ -0,0 +1,9 @@ +@echo off +title Lucy (Dev) +cd /d "%~dp0" +echo Starte Lucy ... (dieses Fenster offen lassen; schliessen = Lucy beenden) +echo. +call npm run dev +echo. +echo Lucy wurde beendet. Taste druecken zum Schliessen. +pause >nul diff --git a/client/lucy-desktop/tsconfig.json b/client/lucy-desktop/tsconfig.json new file mode 100644 index 0000000..90dc141 --- /dev/null +++ b/client/lucy-desktop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": ["node", "electron-vite/node"] + }, + "include": ["src"] +}