Lucy-TTS/F5: Skripte + Batches versionieren, schwere Assets ignoriert
- pocket_server.py (Produktions-TTS mit Stimmen-Waechter), text_norm, Bench-/Diag-Skripte - lucy-f5: f5_server/f5_test/bench_dml (DirectML-Experiment, Phase C/D offen) - .gitignore: venvs/Modelle/Audio/Logs der beiden Ordner + box_recon/gemma_swap-Scratch Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Hitonabi
@@ -0,0 +1,268 @@
|
||||
import re
|
||||
import site
|
||||
import time
|
||||
import jieba
|
||||
import torch
|
||||
import onnxruntime
|
||||
import soundfile as sf
|
||||
import numpy as np
|
||||
from pydub import AudioSegment
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
python_package_path = site.getsitepackages()[-1]
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-f5"
|
||||
vocab_path = BASE + r"\vocab_v1.txt"
|
||||
onnx_model_A = BASE + r"\onnx_f32\F5_Preprocess.onnx"
|
||||
onnx_model_B = BASE + r"\onnx_f32\F5_Transformer.onnx"
|
||||
onnx_model_C = BASE + r"\onnx_f32\F5_Decode.onnx"
|
||||
generated_audio = BASE + r"\bench_out.wav"
|
||||
test_in_english = True # eingebaute engl. Referenz -> kein Chinesisch/Pinyin; Tempo ist sprachunabhängig
|
||||
|
||||
if test_in_english:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_en.wav"
|
||||
ref_text = "Some call me nature, others call me mother nature."
|
||||
# realistischer Satz-Längen-Benchmark (wie eine echte Lucy-Antwort):
|
||||
gen_text = "Of course, Commander. I will restart the service, check the logs, and let you know once everything is running again."
|
||||
else:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_zh.wav" # The reference audio path.
|
||||
ref_text = "对，这就是我，万人敬仰的太乙真人。" # The ASR result of reference audio.
|
||||
gen_text = "对，这就是我，万人敬仰的大可奇奇。" # The target TTS.
|
||||
|
||||
|
||||
import os as _os
|
||||
ORT_Accelerate_Providers = [_os.environ.get("BENCH_PROVIDER", "DmlExecutionProvider")] # DML (9070 XT) oder CPUExecutionProvider
|
||||
# else keep empty.
|
||||
RANDOM_SEED = 9527 # Set seed to reproduce the generated audio
|
||||
NFE_STEP = int(_os.environ.get("BENCH_NFE", "32")) if (_os := __import__("os")) else 32 # via env testbar
|
||||
FUSE_NFE = 1 # Maintain the same values as the exported model.
|
||||
SPEED = 1.0 # Set for talking speed. Only works with dynamic_axes=True
|
||||
MAX_THREADS = 8 # Max CPU parallel threads.
|
||||
DEVICE_ID = 0 # The GPU id, default to 0.
|
||||
MODEL_SAMPLE_RATE = 24000 # Do not modify it.
|
||||
HOP_LENGTH = 256 # It affects the generated audio length and speech speed.
|
||||
|
||||
if "OpenVINOExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_type': 'CPU', # [CPU, NPU, GPU, GPU.0, GPU.1]]
|
||||
'precision': 'ACCURACY', # [FP32, FP16, ACCURACY]
|
||||
'num_of_threads': MAX_THREADS,
|
||||
'num_streams': 1,
|
||||
'enable_opencl_throttling': True,
|
||||
'enable_qdq_optimizer': False # Enable it carefully
|
||||
}
|
||||
]
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_id': DEVICE_ID,
|
||||
'gpu_mem_limit': 8 * 1024 * 1024 * 1024, # 8 GB
|
||||
'arena_extend_strategy': 'kNextPowerOfTwo',
|
||||
'cudnn_conv_algo_search': 'EXHAUSTIVE',
|
||||
'cudnn_conv_use_max_workspace': '1',
|
||||
'do_copy_in_default_stream': '1',
|
||||
'cudnn_conv1d_pad_to_nc1d': '1',
|
||||
'enable_cuda_graph': '0', # Set to '0' to avoid potential errors when enabled.
|
||||
'use_tf32': '0'
|
||||
}
|
||||
]
|
||||
else:
|
||||
# Please config by yourself for others providers.
|
||||
provider_options = None
|
||||
|
||||
|
||||
with open(vocab_path, "r", encoding="utf-8") as f:
|
||||
vocab_char_map = {}
|
||||
for i, char in enumerate(f):
|
||||
vocab_char_map[char[:-1]] = i
|
||||
vocab_size = len(vocab_char_map)
|
||||
|
||||
|
||||
# From the official code
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50) # CRITICAL
|
||||
jieba.initialize()
|
||||
|
||||
final_text_list = []
|
||||
custom_trans = str.maketrans(
|
||||
{";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"}
|
||||
) # add custom trans here, to address oov
|
||||
|
||||
def is_chinese(c):
|
||||
return (
|
||||
"\u3100" <= c <= "\u9fff" # common chinese characters
|
||||
)
|
||||
|
||||
for text in text_list:
|
||||
char_list = []
|
||||
text = text.translate(custom_trans)
|
||||
for seg in jieba.cut(text):
|
||||
seg_byte_len = len(bytes(seg, "UTF-8"))
|
||||
if seg_byte_len == len(seg): # if pure alphabets and symbols
|
||||
if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
|
||||
char_list.append(" ")
|
||||
char_list.extend(seg)
|
||||
elif polyphone and seg_byte_len == 3 * len(seg): # if pure east asian characters
|
||||
seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.append(seg_[i])
|
||||
else: # if mixed characters, alphabets and symbols
|
||||
for c in seg:
|
||||
if ord(c) < 256:
|
||||
char_list.extend(c)
|
||||
elif is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else:
|
||||
char_list.append(c)
|
||||
final_text_list.append(char_list)
|
||||
return final_text_list
|
||||
|
||||
|
||||
# From the official code
|
||||
def list_str_to_idx(
|
||||
text: list[str] | list[list[str]],
|
||||
vocab_char_map: dict[str, int], # {char: idx}
|
||||
padding_value=-1
|
||||
):
|
||||
get_idx = vocab_char_map.get
|
||||
list_idx_tensors = [torch.tensor([get_idx(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
text = torch.nn.utils.rnn.pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_to_int16(audio):
|
||||
max_val = np.max(np.abs(audio))
|
||||
scaling_factor = 32767.0 / max_val if max_val > 0 else 1.0
|
||||
return (audio * float(scaling_factor)).astype(np.int16)
|
||||
|
||||
|
||||
# ONNX Runtime settings
|
||||
onnxruntime.set_seed(RANDOM_SEED)
|
||||
session_opts = onnxruntime.SessionOptions()
|
||||
session_opts.log_severity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.log_verbosity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.inter_op_num_threads = MAX_THREADS # Run different nodes with num_threads. Set 0 for auto.
|
||||
session_opts.intra_op_num_threads = MAX_THREADS # Under the node, execute the operators with num_threads. Set 0 for auto.
|
||||
session_opts.enable_cpu_mem_arena = True # True for execute speed; False for less memory usage.
|
||||
session_opts.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
session_opts.add_session_config_entry("session.intra_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.inter_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.set_denormal_as_zero", "1")
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_A = onnxruntime.InferenceSession(onnx_model_A, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
model_type = ort_session_A._inputs_meta[0].type
|
||||
in_name_A = ort_session_A.get_inputs()
|
||||
out_name_A = ort_session_A.get_outputs()
|
||||
in_name_A0 = in_name_A[0].name
|
||||
in_name_A1 = in_name_A[1].name
|
||||
in_name_A2 = in_name_A[2].name
|
||||
out_name_A0 = out_name_A[0].name
|
||||
out_name_A1 = out_name_A[1].name
|
||||
out_name_A2 = out_name_A[2].name
|
||||
out_name_A3 = out_name_A[3].name
|
||||
out_name_A4 = out_name_A[4].name
|
||||
out_name_A5 = out_name_A[5].name
|
||||
out_name_A6 = out_name_A[6].name
|
||||
out_name_A7 = out_name_A[7].name
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
else:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=ORT_Accelerate_Providers, provider_options=provider_options)
|
||||
ORT_Accelerate_Providers = ort_session_B.get_providers()[0]
|
||||
# For Windows DirectML + Intel/AMD/Nvidia GPU,
|
||||
# pip install onnxruntime-directml --upgrade
|
||||
# ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=['DmlExecutionProvider'])
|
||||
print(f"\nUsable Providers: {ORT_Accelerate_Providers}")
|
||||
model_dtype = ort_session_B._inputs_meta[0].type
|
||||
in_name_B = ort_session_B.get_inputs()
|
||||
out_name_B = ort_session_B.get_outputs()
|
||||
in_name_B0 = in_name_B[0].name
|
||||
in_name_B1 = in_name_B[1].name
|
||||
in_name_B2 = in_name_B[2].name
|
||||
in_name_B3 = in_name_B[3].name
|
||||
in_name_B4 = in_name_B[4].name
|
||||
in_name_B5 = in_name_B[5].name
|
||||
in_name_B6 = in_name_B[6].name
|
||||
in_name_B7 = in_name_B[7].name
|
||||
out_name_B0 = out_name_B[0].name
|
||||
out_name_B1 = out_name_B[1].name
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_C = onnxruntime.InferenceSession(onnx_model_C, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
in_name_C = ort_session_C.get_inputs()
|
||||
out_name_C = ort_session_C.get_outputs()
|
||||
in_name_C0 = in_name_C[0].name
|
||||
in_name_C1 = in_name_C[1].name
|
||||
out_name_C0 = out_name_C[0].name
|
||||
|
||||
# Load the input audio
|
||||
print(f"\nReference Audio: {reference_audio}")
|
||||
audio = np.array(AudioSegment.from_file(reference_audio).set_channels(1).set_frame_rate(MODEL_SAMPLE_RATE).get_array_of_samples(), dtype=np.float32)
|
||||
audio = normalize_to_int16(audio)
|
||||
audio_len = len(audio)
|
||||
audio = audio.reshape(1, 1, -1)
|
||||
|
||||
zh_pause_punc = r"。，、；：？！"
|
||||
ref_text_len = len(ref_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, ref_text))
|
||||
gen_text_len = len(gen_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, gen_text))
|
||||
ref_audio_len = audio_len // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / SPEED)], dtype=np.int64)
|
||||
gen_text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(gen_text, vocab_char_map).numpy()
|
||||
time_step = np.array([0], dtype=np.int32)
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
device_type = 'cpu'
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers or "TensorrtExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'cuda'
|
||||
elif "DmlExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'dml'
|
||||
else:
|
||||
device_type = None
|
||||
|
||||
def run_pipeline():
|
||||
a_out = ort_session_A.run(
|
||||
[out_name_A0, out_name_A1, out_name_A2, out_name_A3, out_name_A4, out_name_A5, out_name_A6, out_name_A7],
|
||||
{in_name_A0: audio, in_name_A1: text_ids, in_name_A2: max_duration})
|
||||
noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ref_signal_len = a_out
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
if device_type:
|
||||
inputs = [onnxruntime.OrtValue.ortvalue_from_numpy(x, device_type, DEVICE_ID) for x in
|
||||
(noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ts)]
|
||||
outputs = [inputs[0], inputs[-1]]
|
||||
iob = ort_session_B.io_binding()
|
||||
for i in range(len(inputs)):
|
||||
iob.bind_ortvalue_input(name=in_name_B[i].name, ortvalue=inputs[i])
|
||||
for i in range(len(outputs)):
|
||||
iob.bind_ortvalue_output(name=out_name_B[i].name, ortvalue=outputs[i])
|
||||
for _ in range(0, NFE_STEP, FUSE_NFE):
|
||||
ort_session_B.run_with_iobinding(iob)
|
||||
noise = onnxruntime.OrtValue.numpy(iob.get_outputs()[0])
|
||||
else:
|
||||
for _ in range(0, NFE_STEP - 1, FUSE_NFE):
|
||||
noise, ts = ort_session_B.run(
|
||||
[out_name_B0, out_name_B1],
|
||||
{in_name_B0: noise, in_name_B1: rope_cos_q, in_name_B2: rope_sin_q, in_name_B3: rope_cos_k,
|
||||
in_name_B4: rope_sin_k, in_name_B5: cat_mel_text, in_name_B6: cat_mel_text_drop, in_name_B7: ts})
|
||||
return ort_session_C.run([out_name_C0], {in_name_C0: noise, in_name_C1: ref_signal_len})[0]
|
||||
|
||||
print(f"\nProvider={ORT_Accelerate_Providers} device_type={device_type} NFE={NFE_STEP}")
|
||||
print("Warmup (DML kompiliert beim 1. Lauf die Shader) ...")
|
||||
t0 = time.time(); _ = run_pipeline(); print(f" warmup gen = {time.time()-t0:.2f}s")
|
||||
best = 1e9
|
||||
for k in range(2):
|
||||
t0 = time.time(); gen = run_pipeline(); dt = time.time() - t0; best = min(best, dt)
|
||||
print(f" run {k+1}: gen = {dt:.2f}s")
|
||||
audio_s = gen.reshape(-1).shape[0] / MODEL_SAMPLE_RATE
|
||||
sf.write(generated_audio, gen.reshape(-1), MODEL_SAMPLE_RATE, format='WAVEX')
|
||||
rtf = best / max(audio_s, 0.01)
|
||||
print(f"\n=== ERGEBNIS NFE={NFE_STEP} === audio={audio_s:.2f}s gen(best)={best:.2f}s RTF={rtf:.2f} "
|
||||
f"({'REAL-TIME' if rtf < 1 else 'zu langsam'})")
|
||||
@@ -0,0 +1,298 @@
|
||||
import re
|
||||
import site
|
||||
import time
|
||||
import jieba
|
||||
import torch
|
||||
import onnxruntime
|
||||
import soundfile as sf
|
||||
import numpy as np
|
||||
from pydub import AudioSegment
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
python_package_path = site.getsitepackages()[-1]
|
||||
|
||||
vocab_path = "/home/DakeQQ/Downloads/F5TTS_v1_Base/vocab.txt" # The F5-TTS model vocab download path. URL: https://huggingface.co/SWivid/F5-TTS/tree/main/F5TTS_v1_Base
|
||||
onnx_model_A = "/home/DakeQQ/Downloads/F5_Optimized/F5_Preprocess.onnx" # The exported onnx model path.
|
||||
onnx_model_B = "/home/DakeQQ/Downloads/F5_Optimized/F5_Transformer.onnx" # The exported onnx model path.
|
||||
onnx_model_C = "/home/DakeQQ/Downloads/F5_Optimized/F5_Decode.onnx" # The exported onnx model path.
|
||||
generated_audio = "./generated_audio.wav"
|
||||
test_in_english = False
|
||||
|
||||
if test_in_english:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_en.wav"
|
||||
ref_text = "Some call me nature, others call me mother nature."
|
||||
gen_text = "Some call me Dake, others call me QQ."
|
||||
else:
|
||||
reference_audio = python_package_path + "/f5_tts/infer/examples/basic/basic_ref_zh.wav" # The reference audio path.
|
||||
ref_text = "对，这就是我，万人敬仰的太乙真人。" # The ASR result of reference audio.
|
||||
gen_text = "对，这就是我，万人敬仰的大可奇奇。" # The target TTS.
|
||||
|
||||
|
||||
ORT_Accelerate_Providers = ['CPUExecutionProvider'] # If you have accelerate devices for : ['CUDAExecutionProvider', 'TensorrtExecutionProvider', 'CoreMLExecutionProvider', 'DmlExecutionProvider', 'OpenVINOExecutionProvider', 'ROCMExecutionProvider', 'MIGraphXExecutionProvider', 'AzureExecutionProvider']
|
||||
# else keep empty.
|
||||
RANDOM_SEED = 9527 # Set seed to reproduce the generated audio
|
||||
NFE_STEP = 32 # F5-TTS model setting, 0~31
|
||||
FUSE_NFE = 1 # Maintain the same values as the exported model.
|
||||
SPEED = 1.0 # Set for talking speed. Only works with dynamic_axes=True
|
||||
MAX_THREADS = 8 # Max CPU parallel threads.
|
||||
DEVICE_ID = 0 # The GPU id, default to 0.
|
||||
MODEL_SAMPLE_RATE = 24000 # Do not modify it.
|
||||
HOP_LENGTH = 256 # It affects the generated audio length and speech speed.
|
||||
|
||||
if "OpenVINOExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_type': 'CPU', # [CPU, NPU, GPU, GPU.0, GPU.1]]
|
||||
'precision': 'ACCURACY', # [FP32, FP16, ACCURACY]
|
||||
'num_of_threads': MAX_THREADS,
|
||||
'num_streams': 1,
|
||||
'enable_opencl_throttling': True,
|
||||
'enable_qdq_optimizer': False # Enable it carefully
|
||||
}
|
||||
]
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers:
|
||||
provider_options = [
|
||||
{
|
||||
'device_id': DEVICE_ID,
|
||||
'gpu_mem_limit': 8 * 1024 * 1024 * 1024, # 8 GB
|
||||
'arena_extend_strategy': 'kNextPowerOfTwo',
|
||||
'cudnn_conv_algo_search': 'EXHAUSTIVE',
|
||||
'cudnn_conv_use_max_workspace': '1',
|
||||
'do_copy_in_default_stream': '1',
|
||||
'cudnn_conv1d_pad_to_nc1d': '1',
|
||||
'enable_cuda_graph': '0', # Set to '0' to avoid potential errors when enabled.
|
||||
'use_tf32': '0'
|
||||
}
|
||||
]
|
||||
else:
|
||||
# Please config by yourself for others providers.
|
||||
provider_options = None
|
||||
|
||||
|
||||
with open(vocab_path, "r", encoding="utf-8") as f:
|
||||
vocab_char_map = {}
|
||||
for i, char in enumerate(f):
|
||||
vocab_char_map[char[:-1]] = i
|
||||
vocab_size = len(vocab_char_map)
|
||||
|
||||
|
||||
# From the official code
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50) # CRITICAL
|
||||
jieba.initialize()
|
||||
|
||||
final_text_list = []
|
||||
custom_trans = str.maketrans(
|
||||
{";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"}
|
||||
) # add custom trans here, to address oov
|
||||
|
||||
def is_chinese(c):
|
||||
return (
|
||||
"\u3100" <= c <= "\u9fff" # common chinese characters
|
||||
)
|
||||
|
||||
for text in text_list:
|
||||
char_list = []
|
||||
text = text.translate(custom_trans)
|
||||
for seg in jieba.cut(text):
|
||||
seg_byte_len = len(bytes(seg, "UTF-8"))
|
||||
if seg_byte_len == len(seg): # if pure alphabets and symbols
|
||||
if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
|
||||
char_list.append(" ")
|
||||
char_list.extend(seg)
|
||||
elif polyphone and seg_byte_len == 3 * len(seg): # if pure east asian characters
|
||||
seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.append(seg_[i])
|
||||
else: # if mixed characters, alphabets and symbols
|
||||
for c in seg:
|
||||
if ord(c) < 256:
|
||||
char_list.extend(c)
|
||||
elif is_chinese(c):
|
||||
char_list.append(" ")
|
||||
char_list.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else:
|
||||
char_list.append(c)
|
||||
final_text_list.append(char_list)
|
||||
return final_text_list
|
||||
|
||||
|
||||
# From the official code
|
||||
def list_str_to_idx(
|
||||
text: list[str] | list[list[str]],
|
||||
vocab_char_map: dict[str, int], # {char: idx}
|
||||
padding_value=-1
|
||||
):
|
||||
get_idx = vocab_char_map.get
|
||||
list_idx_tensors = [torch.tensor([get_idx(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
text = torch.nn.utils.rnn.pad_sequence(list_idx_tensors, padding_value=padding_value, batch_first=True)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_to_int16(audio):
|
||||
max_val = np.max(np.abs(audio))
|
||||
scaling_factor = 32767.0 / max_val if max_val > 0 else 1.0
|
||||
return (audio * float(scaling_factor)).astype(np.int16)
|
||||
|
||||
|
||||
# ONNX Runtime settings
|
||||
onnxruntime.set_seed(RANDOM_SEED)
|
||||
session_opts = onnxruntime.SessionOptions()
|
||||
session_opts.log_severity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.log_verbosity_level = 4 # fatal level = 4, it an adjustable value.
|
||||
session_opts.inter_op_num_threads = MAX_THREADS # Run different nodes with num_threads. Set 0 for auto.
|
||||
session_opts.intra_op_num_threads = MAX_THREADS # Under the node, execute the operators with num_threads. Set 0 for auto.
|
||||
session_opts.enable_cpu_mem_arena = True # True for execute speed; False for less memory usage.
|
||||
session_opts.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
session_opts.add_session_config_entry("session.intra_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.inter_op.allow_spinning", "1")
|
||||
session_opts.add_session_config_entry("session.set_denormal_as_zero", "1")
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_A = onnxruntime.InferenceSession(onnx_model_A, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
model_type = ort_session_A._inputs_meta[0].type
|
||||
in_name_A = ort_session_A.get_inputs()
|
||||
out_name_A = ort_session_A.get_outputs()
|
||||
in_name_A0 = in_name_A[0].name
|
||||
in_name_A1 = in_name_A[1].name
|
||||
in_name_A2 = in_name_A[2].name
|
||||
out_name_A0 = out_name_A[0].name
|
||||
out_name_A1 = out_name_A[1].name
|
||||
out_name_A2 = out_name_A[2].name
|
||||
out_name_A3 = out_name_A[3].name
|
||||
out_name_A4 = out_name_A[4].name
|
||||
out_name_A5 = out_name_A[5].name
|
||||
out_name_A6 = out_name_A[6].name
|
||||
out_name_A7 = out_name_A[7].name
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
else:
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=ORT_Accelerate_Providers, provider_options=provider_options)
|
||||
ORT_Accelerate_Providers = ort_session_B.get_providers()[0]
|
||||
# For Windows DirectML + Intel/AMD/Nvidia GPU,
|
||||
# pip install onnxruntime-directml --upgrade
|
||||
# ort_session_B = onnxruntime.InferenceSession(onnx_model_B, sess_options=session_opts, providers=['DmlExecutionProvider'])
|
||||
print(f"\nUsable Providers: {ORT_Accelerate_Providers}")
|
||||
model_dtype = ort_session_B._inputs_meta[0].type
|
||||
in_name_B = ort_session_B.get_inputs()
|
||||
out_name_B = ort_session_B.get_outputs()
|
||||
in_name_B0 = in_name_B[0].name
|
||||
in_name_B1 = in_name_B[1].name
|
||||
in_name_B2 = in_name_B[2].name
|
||||
in_name_B3 = in_name_B[3].name
|
||||
in_name_B4 = in_name_B[4].name
|
||||
in_name_B5 = in_name_B[5].name
|
||||
in_name_B6 = in_name_B[6].name
|
||||
in_name_B7 = in_name_B[7].name
|
||||
out_name_B0 = out_name_B[0].name
|
||||
out_name_B1 = out_name_B[1].name
|
||||
|
||||
session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
ort_session_C = onnxruntime.InferenceSession(onnx_model_C, sess_options=session_opts, providers=['CPUExecutionProvider'], provider_options=None)
|
||||
in_name_C = ort_session_C.get_inputs()
|
||||
out_name_C = ort_session_C.get_outputs()
|
||||
in_name_C0 = in_name_C[0].name
|
||||
in_name_C1 = in_name_C[1].name
|
||||
out_name_C0 = out_name_C[0].name
|
||||
|
||||
# Load the input audio
|
||||
print(f"\nReference Audio: {reference_audio}")
|
||||
audio = np.array(AudioSegment.from_file(reference_audio).set_channels(1).set_frame_rate(MODEL_SAMPLE_RATE).get_array_of_samples(), dtype=np.float32)
|
||||
audio = normalize_to_int16(audio)
|
||||
audio_len = len(audio)
|
||||
audio = audio.reshape(1, 1, -1)
|
||||
|
||||
zh_pause_punc = r"。，、；：？！"
|
||||
ref_text_len = len(ref_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, ref_text))
|
||||
gen_text_len = len(gen_text.encode('utf-8')) + 3 * len(re.findall(zh_pause_punc, gen_text))
|
||||
ref_audio_len = audio_len // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / SPEED)], dtype=np.int64)
|
||||
gen_text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(gen_text, vocab_char_map).numpy()
|
||||
time_step = np.array([0], dtype=np.int32)
|
||||
|
||||
if "CPUExecutionProvider" in ORT_Accelerate_Providers or not ORT_Accelerate_Providers:
|
||||
device_type = 'cpu'
|
||||
elif "CUDAExecutionProvider" in ORT_Accelerate_Providers or "TensorrtExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'cuda'
|
||||
elif "DmlExecutionProvider" in ORT_Accelerate_Providers:
|
||||
device_type = 'dml'
|
||||
else:
|
||||
device_type = None
|
||||
|
||||
print("\n\nRun F5-TTS by ONNX Runtime.")
|
||||
start_count = time.time()
|
||||
noise, rope_cos_q, rope_sin_q, rope_cos_k, rope_sin_k, cat_mel_text, cat_mel_text_drop, ref_signal_len = ort_session_A.run(
|
||||
[out_name_A0, out_name_A1, out_name_A2, out_name_A3, out_name_A4, out_name_A5, out_name_A6, out_name_A7],
|
||||
{
|
||||
in_name_A0: audio,
|
||||
in_name_A1: text_ids,
|
||||
in_name_A2: max_duration
|
||||
})
|
||||
|
||||
if device_type:
|
||||
inputs = [
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(noise, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_cos_q, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_sin_q, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_cos_k, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(rope_sin_k, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(cat_mel_text, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(cat_mel_text_drop, device_type, DEVICE_ID),
|
||||
onnxruntime.OrtValue.ortvalue_from_numpy(time_step, device_type, DEVICE_ID)
|
||||
]
|
||||
outputs = [
|
||||
inputs[0],
|
||||
inputs[-1]
|
||||
]
|
||||
|
||||
io_binding = ort_session_B.io_binding()
|
||||
for i in range(len(inputs)):
|
||||
io_binding.bind_ortvalue_input(
|
||||
name=in_name_B[i].name,
|
||||
ortvalue=inputs[i]
|
||||
)
|
||||
for i in range(len(outputs)):
|
||||
io_binding.bind_ortvalue_output(
|
||||
name=out_name_B[i].name,
|
||||
ortvalue=outputs[i]
|
||||
)
|
||||
|
||||
print("NFE_STEP: 0")
|
||||
for i in range(0, NFE_STEP, FUSE_NFE):
|
||||
ort_session_B.run_with_iobinding(io_binding)
|
||||
print(f"NFE_STEP: {i + FUSE_NFE}")
|
||||
noise = onnxruntime.OrtValue.numpy(io_binding.get_outputs()[0])
|
||||
else:
|
||||
print("NFE_STEP: 0")
|
||||
for i in range(0, NFE_STEP - 1, FUSE_NFE):
|
||||
noise, time_step = ort_session_B.run(
|
||||
[out_name_B0, out_name_B1],
|
||||
{
|
||||
in_name_B0: noise,
|
||||
in_name_B1: rope_cos_q,
|
||||
in_name_B2: rope_sin_q,
|
||||
in_name_B3: rope_cos_k,
|
||||
in_name_B4: rope_sin_k,
|
||||
in_name_B5: cat_mel_text,
|
||||
in_name_B6: cat_mel_text_drop,
|
||||
in_name_B7: time_step
|
||||
})
|
||||
print(f"NFE_STEP: {i + FUSE_NFE}")
|
||||
|
||||
generated_signal = ort_session_C.run(
|
||||
[out_name_C0],
|
||||
{
|
||||
in_name_C0: noise,
|
||||
in_name_C1: ref_signal_len
|
||||
})[0]
|
||||
end_count = time.time()
|
||||
|
||||
# Save to audio
|
||||
sf.write(generated_audio, generated_signal.reshape(-1), MODEL_SAMPLE_RATE, format='WAVEX')
|
||||
print(f"\nAudio generation is complete.\n\nONNXRuntime Time Cost in Seconds:\n{end_count - start_count:.3f}")
|
||||
@@ -0,0 +1,196 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Lucy-Stimme v2: F5-TTS (deutsch) via ONNX Runtime + DirectML (9070 XT, nativ Windows, kein ROCm).
|
||||
Non-autoregressiv -> keine Kollaps-/Wiederhol-/Männerstimmen-Fehler wie pocket. Satzweises Streaming.
|
||||
Modelliert nach dem verifizierten DML-Benchmark (bench_dml.py) + Export_F5-Preprocessing + pocket_server-Struktur."""
|
||||
import os, io, re, time, threading, logging
|
||||
import numpy as np, soundfile as sf, librosa, jieba, torch
|
||||
import onnxruntime as ort
|
||||
from pypinyin import lazy_pinyin, Style
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import Response, JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
log = logging.getLogger("lucy-f5"); logging.basicConfig(level=logging.INFO)
|
||||
BASE = os.path.dirname(os.path.abspath(__file__))
|
||||
ONNX_DIR = os.environ.get("LUCY_F5_ONNX", os.path.join(BASE, "onnx_de"))
|
||||
VOCAB = os.environ.get("LUCY_F5_VOCAB", os.path.join(BASE, "vocab.txt"))
|
||||
REF_WAV = os.environ.get("LUCY_F5_REF", os.path.join(BASE, "lucy_ref.wav"))
|
||||
REF_TXT = os.environ.get("LUCY_F5_REF_TXT", os.path.join(BASE, "lucy_ref.txt"))
|
||||
PROVIDER = os.environ.get("LUCY_F5_PROVIDER", "DmlExecutionProvider")
|
||||
NFE_STEP = int(os.environ.get("LUCY_F5_NFE", "32")) # MUSS zum Export passen (Zeitplan ist eingebacken)
|
||||
TARGET_RMS = float(os.environ.get("LUCY_TARGET_RMS", "0.09"))
|
||||
SR = 24000; HOP_LENGTH = 256
|
||||
STATE, LOCK = {}, threading.Lock()
|
||||
|
||||
# ---- Text-Preprocessing (aus Export_F5.py; für Deutsch laufen Nicht-CJK-Zeichen einfach durch) ----
|
||||
def _load_vocab(path):
|
||||
m = {}
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for i, ch in enumerate(f):
|
||||
m[ch[:-1]] = i
|
||||
return m
|
||||
|
||||
def convert_char_to_pinyin(text_list, polyphone=True):
|
||||
if jieba.dt.initialized is False:
|
||||
jieba.default_logger.setLevel(50); jieba.initialize()
|
||||
out, trans = [], str.maketrans({";": ",", "“": '"', "”": '"', "‘": "'", "’": "'"})
|
||||
def is_zh(c): return "" <= c <= "鿿"
|
||||
for text in text_list:
|
||||
cl = []; text = text.translate(trans)
|
||||
for seg in jieba.cut(text):
|
||||
blen = len(bytes(seg, "UTF-8"))
|
||||
if blen == len(seg):
|
||||
if cl and blen > 1 and cl[-1] not in " :'\"": cl.append(" ")
|
||||
cl.extend(seg)
|
||||
elif polyphone and blen == 3 * len(seg):
|
||||
pin = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
|
||||
for i, c in enumerate(seg):
|
||||
if is_zh(c): cl.append(" ")
|
||||
cl.append(pin[i])
|
||||
else:
|
||||
for c in seg:
|
||||
if ord(c) < 256: cl.extend(c)
|
||||
elif is_zh(c): cl.append(" "); cl.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
|
||||
else: cl.append(c)
|
||||
out.append(cl)
|
||||
return out
|
||||
|
||||
def list_str_to_idx(text, vocab_map, padding_value=-1):
|
||||
get = vocab_map.get
|
||||
tensors = [torch.tensor([get(c, 0) for c in t], dtype=torch.int32) for t in text]
|
||||
return torch.nn.utils.rnn.pad_sequence(tensors, padding_value=padding_value, batch_first=True).numpy()
|
||||
|
||||
_ZH_PUNC = r"。，、；：？！"
|
||||
def _text_len(s): return len(s.encode("utf-8")) + 3 * len(re.findall(_ZH_PUNC, s))
|
||||
|
||||
# ---- Satz-Splitter (wie pocket) ----
|
||||
_SENT_RX = re.compile(r".+?(?:[.!?…]+(?:\s|$)|$)", re.S)
|
||||
def split_sentences(text, min_len=30):
|
||||
parts = [m.group(0).strip() for m in _SENT_RX.finditer(text.strip())]
|
||||
out = []
|
||||
for p in parts:
|
||||
if not p: continue
|
||||
if out and len(out[-1]) < min_len: out[-1] = f"{out[-1]} {p}"
|
||||
else: out.append(p)
|
||||
return out or [text.strip()]
|
||||
|
||||
def cleanup(a, sr):
|
||||
"""F5-Output putzen: Stille-Trim hinten, RMS-Norm auf TARGET_RMS, Peak-Clamp, 80ms-Pads."""
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
if a.size == 0: return a
|
||||
rev, _ = librosa.effects.trim(a[::-1], top_db=40); a = rev[::-1] if rev.size else a
|
||||
yt, _ = librosa.effects.trim(a, top_db=40); a = yt if yt.size else a
|
||||
rms = float(np.sqrt(np.mean(a ** 2))) or 1e-9
|
||||
a = a * (TARGET_RMS / rms)
|
||||
peak = float(np.max(np.abs(a)))
|
||||
if peak > 0.95: a = a * (0.95 / peak)
|
||||
fi = min(int(0.008 * sr), a.size // 2)
|
||||
if fi > 0:
|
||||
a[:fi] *= np.linspace(0., 1., fi, dtype=np.float32); a[-fi:] *= np.linspace(1., 0., fi, dtype=np.float32)
|
||||
pad = np.zeros(int(0.08 * sr), dtype=np.float32)
|
||||
return np.concatenate([pad, a, pad])
|
||||
|
||||
def _to_pcm16(a):
|
||||
a = np.asarray(a, dtype=np.float32).reshape(-1)
|
||||
np.clip(a, -0.95, 0.95, out=a)
|
||||
return (a * 32767.0).astype("<i2").tobytes()
|
||||
|
||||
# ---- ONNX-Inferenz (A=Preprocess CPU, B=Transformer DML+io_binding, C=Decode CPU) ----
|
||||
def _infer(gen_text: str) -> np.ndarray:
|
||||
s = STATE
|
||||
ref_text = s["ref_text"]
|
||||
rt_len = _text_len(ref_text); gt_len = max(_text_len(gen_text), 1)
|
||||
ref_audio_len = s["ref_audio"].shape[-1] // HOP_LENGTH + 1
|
||||
max_duration = np.array([ref_audio_len + int(ref_audio_len / rt_len * gt_len)], dtype=np.int64)
|
||||
text = convert_char_to_pinyin([ref_text + gen_text])
|
||||
text_ids = list_str_to_idx(text, s["vocab"])
|
||||
A = s["A"].run(s["A_out"], {s["A_in"][0]: s["ref_audio"], s["A_in"][1]: text_ids, s["A_in"][2]: max_duration})
|
||||
noise, rcq, rsq, rck, rsk, cmt, cmtd, ref_signal_len = A
|
||||
dev = s["dev"]
|
||||
if dev: # DirectML/CUDA: io_binding, Tensoren GPU-resident über die NFE-Schleife
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
ins = [ort.OrtValue.ortvalue_from_numpy(x, dev, 0) for x in (noise, rcq, rsq, rck, rsk, cmt, cmtd, ts)]
|
||||
outs = [ins[0], ins[-1]]
|
||||
iob = s["B"].io_binding()
|
||||
for i in range(len(ins)): iob.bind_ortvalue_input(name=s["B_in"][i], ortvalue=ins[i])
|
||||
for i in range(len(outs)): iob.bind_ortvalue_output(name=s["B_out"][i], ortvalue=outs[i])
|
||||
for _ in range(0, NFE_STEP, 1): s["B"].run_with_iobinding(iob)
|
||||
noise = ort.OrtValue.numpy(iob.get_outputs()[0])
|
||||
else:
|
||||
ts = np.array([0], dtype=np.int32)
|
||||
for _ in range(0, NFE_STEP - 1, 1):
|
||||
noise, ts = s["B"].run(s["B_out"], {s["B_in"][0]: noise, s["B_in"][1]: rcq, s["B_in"][2]: rsq,
|
||||
s["B_in"][3]: rck, s["B_in"][4]: rsk, s["B_in"][5]: cmt, s["B_in"][6]: cmtd, s["B_in"][7]: ts})
|
||||
out = s["C"].run([s["C_out"]], {s["C_in"][0]: noise, s["C_in"][1]: ref_signal_len})[0]
|
||||
a = np.asarray(out).reshape(-1).astype(np.float32)
|
||||
if a.dtype != np.float32 or np.max(np.abs(a)) > 1.5: # int16-Decoder -> auf float
|
||||
a = a / 32768.0
|
||||
return a
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
t0 = time.time(); log.info("Lade F5 ONNX (%s) ...", PROVIDER)
|
||||
so = ort.SessionOptions(); so.log_severity_level = 4
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
A = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Preprocess.onnx"), so, providers=["CPUExecutionProvider"])
|
||||
B = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Transformer.onnx"), so, providers=[PROVIDER])
|
||||
C = ort.InferenceSession(os.path.join(ONNX_DIR, "F5_Decode.onnx"), so, providers=["CPUExecutionProvider"])
|
||||
prov = B.get_providers()[0]
|
||||
dev = "dml" if "Dml" in prov else ("cuda" if "CUDA" in prov or "Tensorrt" in prov else None)
|
||||
# Lucy-Referenz als int16 laden (Decoder/Preprocess erwartet int16-Pfad)
|
||||
ref, _sr = sf.read(REF_WAV, dtype="float32", always_2d=False)
|
||||
ref = np.asarray(ref, dtype=np.float32).reshape(-1)
|
||||
if _sr != SR: ref = librosa.resample(ref, orig_sr=_sr, target_sr=SR)
|
||||
mx = np.max(np.abs(ref)) or 1.0
|
||||
ref_i16 = (ref * (32767.0 / mx)).astype(np.int16).reshape(1, 1, -1)
|
||||
STATE.update(
|
||||
A=A, B=B, C=C, dev=dev, prov=prov,
|
||||
A_in=[i.name for i in A.get_inputs()], A_out=[o.name for o in A.get_outputs()],
|
||||
B_in=[i.name for i in B.get_inputs()], B_out=[o.name for o in B.get_outputs()],
|
||||
C_in=[i.name for i in C.get_inputs()], C_out=C.get_outputs()[0].name,
|
||||
vocab=_load_vocab(VOCAB), ref_audio=ref_i16,
|
||||
ref_text=open(REF_TXT, encoding="utf-8").read().strip(),
|
||||
)
|
||||
log.info("Lucy-F5 bereit in %.1fs (Provider=%s, dev=%s, NFE=%d)", time.time() - t0, prov, dev, NFE_STEP)
|
||||
yield
|
||||
STATE.clear()
|
||||
|
||||
app = FastAPI(title="Lucy TTS (F5/DirectML)", lifespan=lifespan)
|
||||
|
||||
class Req(BaseModel):
|
||||
text: str
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok" if "A" in STATE else "loading", "engine": "f5-tts",
|
||||
"provider": STATE.get("prov"), "nfe": NFE_STEP, "sr": SR}
|
||||
|
||||
@app.post("/tts")
|
||||
def tts(req: Req):
|
||||
if "A" not in STATE: return JSONResponse({"error": "loading"}, status_code=503)
|
||||
t0 = time.time()
|
||||
parts = []
|
||||
with LOCK:
|
||||
for sent in split_sentences(req.text):
|
||||
parts.append(cleanup(_infer(sent), SR))
|
||||
a = np.concatenate(parts) if parts else np.zeros(0, np.float32)
|
||||
buf = io.BytesIO(); sf.write(buf, a, SR, format="WAV", subtype="PCM_16"); buf.seek(0)
|
||||
dur = a.size / SR; gen = time.time() - t0
|
||||
log.info("/tts %dZ audio=%.1fs gen=%.1fs rtf=%.2f", len(req.text), dur, gen, gen / max(dur, 0.01))
|
||||
return Response(buf.read(), media_type="audio/wav",
|
||||
headers={"X-Audio-Seconds": f"{dur:.2f}", "X-Gen-Seconds": f"{gen:.2f}"})
|
||||
|
||||
@app.post("/tts/stream")
|
||||
def tts_stream(req: Req):
|
||||
if "A" not in STATE: return JSONResponse({"error": "loading"}, status_code=503)
|
||||
sentences = split_sentences(req.text)
|
||||
def pcm():
|
||||
t0 = time.time(); total = 0; first = True
|
||||
with LOCK:
|
||||
for sent in sentences:
|
||||
a = cleanup(_infer(sent), SR); total += a.size
|
||||
if first: log.info("/tts/stream TTFB=%.2fs (%d Sätze)", time.time() - t0, len(sentences)); first = False
|
||||
yield _to_pcm16(a)
|
||||
log.info("/tts/stream %dZ audio=%.1fs gen=%.1fs", len(req.text), total / SR, time.time() - t0)
|
||||
return StreamingResponse(pcm(), media_type="application/octet-stream", headers={"X-Sample-Rate": str(SR)})
|
||||
@@ -0,0 +1,44 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""F5-TTS Qualitäts-/Tempo-Test (CPU): dt. Finetune + Lucy-Klon. Gleiche Sätze wie pocket -> A/B."""
|
||||
import os, time, soundfile as sf, numpy as np, torch
|
||||
# torchcodec/ffmpeg fehlt -> torchaudio.load/save auf soundfile umbiegen (wie bei OuteTTS-Patch)
|
||||
import torchaudio
|
||||
def _ta_load(path, *a, **k):
|
||||
data, sr = sf.read(str(path), dtype="float32", always_2d=True)
|
||||
return torch.from_numpy(data.T.copy()), sr
|
||||
def _ta_save(path, tensor, sr, *a, **k):
|
||||
arr = np.asarray(tensor.detach().cpu().numpy())
|
||||
sf.write(str(path), arr.T if arr.ndim == 2 else arr, sr)
|
||||
torchaudio.load = _ta_load
|
||||
torchaudio.save = _ta_save
|
||||
from f5_tts.api import F5TTS
|
||||
|
||||
BASE = r"F:\Coding Stuff\mission-control-2\client\lucy-f5"
|
||||
OUT = r"C:\Users\TobisPC\Desktop\lucy_f5_test"; os.makedirs(OUT, exist_ok=True)
|
||||
ref = os.path.join(BASE, "lucy_ref.wav")
|
||||
ref_text = open(os.path.join(BASE, "lucy_ref.txt"), encoding="utf-8").read().strip()
|
||||
print("REF_TEXT:", ref_text[:80], flush=True)
|
||||
|
||||
t0 = time.time()
|
||||
f5 = F5TTS(model="F5TTS_Base",
|
||||
ckpt_file=os.path.join(BASE, "model_f5tts_german.safetensors"),
|
||||
vocab_file=os.path.join(BASE, "vocab.txt"), device="cpu")
|
||||
print(f"Modell geladen in {time.time()-t0:.1f}s (Vocoder evtl. erst geladen)", flush=True)
|
||||
|
||||
SENT = {
|
||||
"kurz": "Hallo Commander, ich höre dich.",
|
||||
"mittel":"Guten Morgen, Commander. Das Backup ist sauber durchgelaufen und es gab keine Fehler.",
|
||||
"lang": "Natürlich kümmere ich mich darum, Commander. Ich starte den Dienst neu, prüfe die Protokolle und melde mich, sobald alles wieder läuft.",
|
||||
}
|
||||
for name, text in SENT.items():
|
||||
t0 = time.time()
|
||||
wav, sr, _ = f5.infer(ref_file=ref, ref_text=ref_text, gen_text=text,
|
||||
nfe_step=32, target_rms=0.1, remove_silence=True)
|
||||
dt = time.time() - t0
|
||||
wav = np.asarray(wav, dtype=np.float32).reshape(-1)
|
||||
peak = float(np.max(np.abs(wav))) # Peak-Limiter gegen Clipping (F5 traf 1.0)
|
||||
if peak > 0.95: wav = wav * (0.95 / peak)
|
||||
sf.write(os.path.join(OUT, f"{name}.wav"), wav, sr)
|
||||
secs = len(wav) / sr
|
||||
print(f"[{name:6}] gen={dt:6.1f}s audio={secs:5.1f}s RTF={dt/max(secs,0.01):5.2f}", flush=True)
|
||||
print("F5_TEST_DONE", flush=True)
|
||||
@@ -0,0 +1 @@
|
||||
Hallo, schön, dass du da bist. Ich bin deine persönliche Assistentin und begleite dich durch deinen Tag.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user