xuning / sherpaonnx

转到一个项目
Toggle navigation Toggle navigation pinning
Fangjun Kuang
Committed by GitHub
c2545049 1 parent 35f5ff31

Add Pascal API for Kokoro TTS 1.0 (#1807)

内嵌 并排对比
正在显示 7 个修改的文件 包含 407 行增加 和 2 行删除
@@ -154,6 +154,12 @@ jobs: @@ -154,6 +154,12 @@ jobs:
154 ls -lh 154 ls -lh
155 echo "---" 155 echo "---"
156 156
  157 + ./run-kokoro-zh-en.sh
  158 + rm -rf kokoro-multi-*
  159 + rm kokoro-zh-en
  160 + ls -lh
  161 + echo "---"
  162 +
157 ./run-kokoro-en.sh 163 ./run-kokoro-en.sh
158 rm -rf kokoro-en-* 164 rm -rf kokoro-en-*
159 rm kokoro-en 165 rm kokoro-en
@@ -8,3 +8,5 @@ matcha-zh-playback @@ -8,3 +8,5 @@ matcha-zh-playback
8 matcha-en-playback 8 matcha-en-playback
9 kokoro-en 9 kokoro-en
10 kokoro-en-playback 10 kokoro-en-playback
  11 +kokoro-zh-en
  12 +kokoro-zh-en-playback
  1 +{ Copyright (c) 2025 Xiaomi Corporation }
  2 +program kokoro_en_playback;
  3 +{
  4 +This file shows how to use the text to speech API of sherpa-onnx
  5 +with Kokoro models (Chinese + English).
  6 +
  7 +It generates speech from text and saves it to a wave file.
  8 +
  9 +Note that it plays the audio back as it is still generating.
  10 +}
  11 +
  12 +{$mode objfpc}
  13 +
  14 +uses
  15 + {$ifdef unix}
  16 + cthreads,
  17 + {$endif}
  18 + SysUtils,
  19 + dos,
  20 + ctypes,
  21 + portaudio,
  22 + sherpa_onnx;
  23 +
  24 +var
  25 + CriticalSection: TRTLCriticalSection;
  26 +
  27 + Tts: TSherpaOnnxOfflineTts;
  28 + Audio: TSherpaOnnxGeneratedAudio;
  29 + Resampler: TSherpaOnnxLinearResampler;
  30 +
  31 + Text: AnsiString;
  32 + Speed: Single = 1.0; {Use a larger value to speak faster}
  33 + SpeakerId: Integer = 47;
  34 + Buffer: TSherpaOnnxCircularBuffer;
  35 + FinishedGeneration: Boolean = False;
  36 + FinishedPlaying: Boolean = False;
  37 +
  38 + Version: String;
  39 + EnvStr: String;
  40 + Status: Integer;
  41 + NumDevices: Integer;
  42 + DeviceIndex: Integer;
  43 + DeviceInfo: PPaDeviceInfo;
  44 +
  45 + { If you get EDivByZero: Division by zero error, please change the sample rate
  46 + to the one supported by your microphone.
  47 + }
  48 + DeviceSampleRate: Integer = 48000;
  49 + I: Integer;
  50 + Param: TPaStreamParameters;
  51 + Stream: PPaStream;
  52 + Wave: TSherpaOnnxWave;
  53 +
  54 +function GenerateCallback(
  55 + Samples: pcfloat; N: cint32;
  56 + Arg: Pointer): cint; cdecl;
  57 +begin
  58 + EnterCriticalSection(CriticalSection);
  59 + try
  60 + if Resampler <> nil then
  61 + Buffer.Push(Resampler.Resample(Samples, N, False))
  62 + else
  63 + Buffer.Push(Samples, N);
  64 + finally
  65 + LeaveCriticalSection(CriticalSection);
  66 + end;
  67 +
  68 + { 1 means to continue generating; 0 means to stop generating. }
  69 + Result := 1;
  70 +end;
  71 +
  72 +function PlayCallback(
  73 + input: Pointer; output: Pointer;
  74 + frameCount: culong;
  75 + timeInfo: PPaStreamCallbackTimeInfo;
  76 + statusFlags: TPaStreamCallbackFlags;
  77 + userData: Pointer ): cint; cdecl;
  78 +var
  79 + Samples: TSherpaOnnxSamplesArray;
  80 + I: Integer;
  81 +begin
  82 + EnterCriticalSection(CriticalSection);
  83 + try
  84 + if Buffer.Size >= frameCount then
  85 + begin
  86 + Samples := Buffer.Get(Buffer.Head, FrameCount);
  87 + Buffer.Pop(FrameCount);
  88 + end
  89 + else if Buffer.Size > 0 then
  90 + begin
  91 + Samples := Buffer.Get(Buffer.Head, Buffer.Size);
  92 + Buffer.Pop(Buffer.Size);
  93 + SetLength(Samples, frameCount);
  94 + end
  95 + else
  96 + SetLength(Samples, frameCount);
  97 +
  98 + for I := 0 to frameCount - 1 do
  99 + pcfloat(output)[I] := Samples[I];
  100 +
  101 + if (Buffer.Size > 0) or (not FinishedGeneration) then
  102 + Result := paContinue
  103 + else
  104 + begin
  105 + Result := paComplete;
  106 + FinishedPlaying := True;
  107 + end;
  108 + finally
  109 + LeaveCriticalSection(CriticalSection);
  110 + end;
  111 +end;
  112 +
  113 +function GetOfflineTts: TSherpaOnnxOfflineTts;
  114 +var
  115 + Config: TSherpaOnnxOfflineTtsConfig;
  116 +begin
  117 + Config.Model.Kokoro.Model := './kokoro-multi-lang-v1_0/model.onnx';
  118 + Config.Model.Kokoro.Voices := './kokoro-multi-lang-v1_0/voices.bin';
  119 + Config.Model.Kokoro.Tokens := './kokoro-multi-lang-v1_0/tokens.txt';
  120 + Config.Model.Kokoro.DataDir := './kokoro-multi-lang-v1_0/espeak-ng-data';
  121 + Config.Model.Kokoro.DictDir := './kokoro-multi-lang-v1_0/dict';
  122 + Config.Model.Kokoro.Lexicon := './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt';
  123 + Config.Model.NumThreads := 2;
  124 + Config.Model.Debug := False;
  125 + Config.MaxNumSentences := 1;
  126 +
  127 + Result := TSherpaOnnxOfflineTts.Create(Config);
  128 +end;
  129 +
  130 +begin
  131 + Tts := GetOfflineTts;
  132 + if Tts.GetSampleRate <> DeviceSampleRate then
  133 + Resampler := TSherpaOnnxLinearResampler.Create(Tts.GetSampleRate, DeviceSampleRate);
  134 +
  135 + Version := String(Pa_GetVersionText);
  136 + WriteLn('Version is ', Version);
  137 + Status := Pa_Initialize;
  138 + if Status <> paNoError then
  139 + begin
  140 + WriteLn('Failed to initialize portaudio, ', Pa_GetErrorText(Status));
  141 + Exit;
  142 + end;
  143 +
  144 + NumDevices := Pa_GetDeviceCount;
  145 + WriteLn('Num devices: ', NumDevices);
  146 +
  147 + DeviceIndex := Pa_GetDefaultOutputDevice;
  148 +
  149 + if DeviceIndex = paNoDevice then
  150 + begin
  151 + WriteLn('No default output device found');
  152 + Pa_Terminate;
  153 + Exit;
  154 + end;
  155 +
  156 + EnvStr := GetEnv('SHERPA_ONNX_MIC_DEVICE');
  157 + if EnvStr <> '' then
  158 + begin
  159 + DeviceIndex := StrToIntDef(EnvStr, DeviceIndex);
  160 + WriteLn('Use device index from environment variable SHERPA_ONNX_MIC_DEVICE: ', EnvStr);
  161 + end;
  162 +
  163 + for I := 0 to (NumDevices - 1) do
  164 + begin
  165 + DeviceInfo := Pa_GetDeviceInfo(I);
  166 + if I = DeviceIndex then
  167 + { WriteLn(Format(' * %d %s', [I, DeviceInfo^.Name])) }
  168 + WriteLn(Format(' * %d %s', [I, AnsiString(DeviceInfo^.Name)]))
  169 + else
  170 + WriteLn(Format(' %d %s', [I, AnsiString(DeviceInfo^.Name)]));
  171 + end;
  172 +
  173 + WriteLn('Use device ', DeviceIndex);
  174 + WriteLn(' Name ', Pa_GetDeviceInfo(DeviceIndex)^.Name);
  175 + WriteLn(' Max output channels ', Pa_GetDeviceInfo(DeviceIndex)^.MaxOutputChannels);
  176 +
  177 + Initialize(Param);
  178 + Param.Device := DeviceIndex;
  179 + Param.ChannelCount := 1;
  180 + Param.SampleFormat := paFloat32;
  181 + param.SuggestedLatency := Pa_GetDeviceInfo(DeviceIndex)^.DefaultHighOutputLatency;
  182 + param.HostApiSpecificStreamInfo := nil;
  183 +
  184 + Buffer := TSherpaOnnxCircularBuffer.Create(30 * DeviceSampleRate);
  185 +
  186 +
  187 + { Note(fangjun): PortAudio invokes PlayCallback in a separate thread. }
  188 + Status := Pa_OpenStream(stream, nil, @Param, DeviceSampleRate, paFramesPerBufferUnspecified, paNoFlag,
  189 + PPaStreamCallback(@PlayCallback), nil);
  190 +
  191 + if Status <> paNoError then
  192 + begin
  193 + WriteLn('Failed to open stream, ', Pa_GetErrorText(Status));
  194 + Pa_Terminate;
  195 + Exit;
  196 + end;
  197 +
  198 + InitCriticalSection(CriticalSection);
  199 +
  200 + Status := Pa_StartStream(stream);
  201 + if Status <> paNoError then
  202 + begin
  203 + WriteLn('Failed to start stream, ', Pa_GetErrorText(Status));
  204 + Pa_Terminate;
  205 + Exit;
  206 + end;
  207 +
  208 + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers');
  209 +
  210 + Text := '中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？';
  211 +
  212 + Audio := Tts.Generate(Text, SpeakerId, Speed,
  213 + PSherpaOnnxGeneratedAudioCallbackWithArg(@GenerateCallback), nil);
  214 + FinishedGeneration := True;
  215 + SherpaOnnxWriteWave('./kokoro-zh-en-playback-47.wav', Audio.Samples, Audio.SampleRate);
  216 + WriteLn('Saved to ./kokoro-zh-en-playback-47.wav');
  217 +
  218 + while not FinishedPlaying do
  219 + Pa_Sleep(100); {sleep for 0.1 second }
  220 + {TODO(fangjun): Use an event to indicate the play is finished}
  221 +
  222 + DoneCriticalSection(CriticalSection);
  223 +
  224 + FreeAndNil(Tts);
  225 + FreeAndNil(Resampler);
  226 +
  227 + Status := Pa_CloseStream(stream);
  228 + if Status <> paNoError then
  229 + begin
  230 + WriteLn('Failed to close stream, ', Pa_GetErrorText(Status));
  231 + Exit;
  232 + end;
  233 +
  234 + Status := Pa_Terminate;
  235 + if Status <> paNoError then
  236 + begin
  237 + WriteLn('Failed to deinitialize portaudio, ', Pa_GetErrorText(Status));
  238 + Exit;
  239 + end;
  240 +end.
  241 +
  1 +{ Copyright (c) 2025 Xiaomi Corporation }
  2 +program kokoro_en;
  3 +{
  4 +This file shows how to use the text to speech API of sherpa-onnx
  5 +with Kokoro TTS models (Chinese + English).
  6 +
  7 +It generates speech from text and saves it to a wave file.
  8 +
  9 +If you want to play it while it is generating, please see
  10 +./kokoro-en-playback.pas
  11 +}
  12 +
  13 +{$mode objfpc}
  14 +
  15 +uses
  16 + SysUtils,
  17 + sherpa_onnx;
  18 +
  19 +function GetOfflineTts: TSherpaOnnxOfflineTts;
  20 +var
  21 + Config: TSherpaOnnxOfflineTtsConfig;
  22 +begin
  23 + Config.Model.Kokoro.Model := './kokoro-multi-lang-v1_0/model.onnx';
  24 + Config.Model.Kokoro.Voices := './kokoro-multi-lang-v1_0/voices.bin';
  25 + Config.Model.Kokoro.Tokens := './kokoro-multi-lang-v1_0/tokens.txt';
  26 + Config.Model.Kokoro.DataDir := './kokoro-multi-lang-v1_0/espeak-ng-data';
  27 + Config.Model.Kokoro.DictDir := './kokoro-multi-lang-v1_0/dict';
  28 + Config.Model.Kokoro.Lexicon := './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt';
  29 + Config.Model.NumThreads := 2;
  30 + Config.Model.Debug := False;
  31 + Config.MaxNumSentences := 1;
  32 +
  33 + Result := TSherpaOnnxOfflineTts.Create(Config);
  34 +end;
  35 +
  36 +var
  37 + Tts: TSherpaOnnxOfflineTts;
  38 + Audio: TSherpaOnnxGeneratedAudio;
  39 +
  40 + Text: AnsiString;
  41 + Speed: Single = 1.0; {Use a larger value to speak faster}
  42 + SpeakerId: Integer = 46;
  43 +
  44 +begin
  45 + Tts := GetOfflineTts;
  46 +
  47 + WriteLn('There are ', Tts.GetNumSpeakers, ' speakers');
  48 +
  49 + Text := '中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？';
  50 +
  51 + Audio := Tts.Generate(Text, SpeakerId, Speed);
  52 + SherpaOnnxWriteWave('./kokoro-zh-en-46.wav', Audio.Samples, Audio.SampleRate);
  53 + WriteLn('Saved to ./kokoro-zh-en-46.wav');
  54 +
  55 + FreeAndNil(Tts);
  56 +end.
  57 +
  1 +#!/usr/bin/env bash
  2 +
  3 +set -ex
  4 +
  5 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
  6 +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd)
  7 +
  8 +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR"
  9 +
  10 +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then
  11 + mkdir -p ../../build
  12 + pushd ../../build
  13 + cmake \
  14 + -DCMAKE_INSTALL_PREFIX=./install \
  15 + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
  16 + -DSHERPA_ONNX_ENABLE_TESTS=OFF \
  17 + -DSHERPA_ONNX_ENABLE_CHECK=OFF \
  18 + -DBUILD_SHARED_LIBS=ON \
  19 + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
  20 + ..
  21 +
  22 + cmake --build . --target install --config Release
  23 + popd
  24 +fi
  25 +
  26 +# please visit
  27 +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html
  28 +if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then
  29 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
  30 + tar xf kokoro-multi-lang-v1_0.tar.bz2
  31 + rm kokoro-multi-lang-v1_0.tar.bz2
  32 +fi
  33 +
  34 +fpc \
  35 + -dSHERPA_ONNX_USE_SHARED_LIBS \
  36 + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \
  37 + -Fl$SHERPA_ONNX_DIR/build/install/lib \
  38 + -Fl/usr/local/Cellar/portaudio/19.7.0/lib \
  39 + ./kokoro-zh-en-playback.pas
  40 +
  41 +# Please see ../portaudio-test/README.md
  42 +# for how to install portaudio on macOS
  43 +
  44 +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH
  45 +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH
  46 +
  47 +./kokoro-zh-en-playback
  1 +#!/usr/bin/env bash
  2 +
  3 +set -ex
  4 +
  5 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
  6 +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd)
  7 +
  8 +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR"
  9 +
  10 +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then
  11 + mkdir -p ../../build
  12 + pushd ../../build
  13 + cmake \
  14 + -DCMAKE_INSTALL_PREFIX=./install \
  15 + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
  16 + -DSHERPA_ONNX_ENABLE_TESTS=OFF \
  17 + -DSHERPA_ONNX_ENABLE_CHECK=OFF \
  18 + -DBUILD_SHARED_LIBS=ON \
  19 + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
  20 + ..
  21 +
  22 + cmake --build . --target install --config Release
  23 + popd
  24 +fi
  25 +
  26 +# please visit
  27 +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html
  28 +if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then
  29 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
  30 + tar xf kokoro-multi-lang-v1_0.tar.bz2
  31 + rm kokoro-multi-lang-v1_0.tar.bz2
  32 +fi
  33 +
  34 +fpc \
  35 + -dSHERPA_ONNX_USE_SHARED_LIBS \
  36 + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \
  37 + -Fl$SHERPA_ONNX_DIR/build/install/lib \
  38 + ./kokoro-zh-en.pas
  39 +
  40 +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH
  41 +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH
  42 +
  43 +./kokoro-zh-en
@@ -82,6 +82,8 @@ type @@ -82,6 +82,8 @@ type
82 Tokens: AnsiString; 82 Tokens: AnsiString;
83 DataDir: AnsiString; 83 DataDir: AnsiString;
84 LengthScale: Single; 84 LengthScale: Single;
  85 + DictDir: AnsiString;
  86 + Lexicon: AnsiString;
85 87
86 function ToString: AnsiString; 88 function ToString: AnsiString;
87 class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig); 89 class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig);
@@ -757,6 +759,8 @@ type @@ -757,6 +759,8 @@ type
757 Tokens: PAnsiChar; 759 Tokens: PAnsiChar;
758 DataDir: PAnsiChar; 760 DataDir: PAnsiChar;
759 LengthScale: cfloat; 761 LengthScale: cfloat;
  762 + DictDir: PAnsiChar;
  763 + Lexicon: PAnsiChar;
760 end; 764 end;
761 765
762 SherpaOnnxOfflineTtsModelConfig = record 766 SherpaOnnxOfflineTtsModelConfig = record
@@ -1931,9 +1935,12 @@ begin @@ -1931,9 +1935,12 @@ begin
1931 'Voices := %s, ' + 1935 'Voices := %s, ' +
1932 'Tokens := %s, ' + 1936 'Tokens := %s, ' +
1933 'DataDir := %s, ' + 1937 'DataDir := %s, ' +
1934 - 'LengthScale := %.2f' + 1938 + 'LengthScale := %.2f, ' +
  1939 + 'DictDir := %s, ' +
  1940 + 'Lexicon := %s' +
1935 ')', 1941 ')',
1936 - [Self.Model, Self.Voices, Self.Tokens, Self.DataDir, Self.LengthScale]); 1942 + [Self.Model, Self.Voices, Self.Tokens, Self.DataDir, Self.LengthScale,
  1943 + Self.DictDir, Self.Lexicon]);
1937 end; 1944 end;
1938 1945
1939 class operator TSherpaOnnxOfflineTtsKokoroModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig); 1946 class operator TSherpaOnnxOfflineTtsKokoroModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKokoroModelConfig);
@@ -2010,6 +2017,8 @@ begin @@ -2010,6 +2017,8 @@ begin
2010 C.Model.Kokoro.Tokens := PAnsiChar(Config.Model.Kokoro.Tokens); 2017 C.Model.Kokoro.Tokens := PAnsiChar(Config.Model.Kokoro.Tokens);
2011 C.Model.Kokoro.DataDir := PAnsiChar(Config.Model.Kokoro.DataDir); 2018 C.Model.Kokoro.DataDir := PAnsiChar(Config.Model.Kokoro.DataDir);
2012 C.Model.Kokoro.LengthScale := Config.Model.Kokoro.LengthScale; 2019 C.Model.Kokoro.LengthScale := Config.Model.Kokoro.LengthScale;
  2020 + C.Model.Kokoro.DictDir := PAnsiChar(Config.Model.Kokoro.DictDir);
  2021 + C.Model.Kokoro.Lexicon := PAnsiChar(Config.Model.Kokoro.Lexicon);
2013 2022
2014 C.Model.NumThreads := Config.Model.NumThreads; 2023 C.Model.NumThreads := Config.Model.NumThreads;
2015 C.Model.Provider := PAnsiChar(Config.Model.Provider); 2024 C.Model.Provider := PAnsiChar(Config.Model.Provider);