V5 / f52bca255516
KokoroTts.cs
Lilith/Core/Kokoro/KokoroTts.cs
Unchanged from the preceding versionCS
using KokoroSharp.Processing;
using System.Text.RegularExpressions;
using KokoroSharp;
using KokoroSharp.Core;
using Lilith.Core.Logging;
using NAudio.Wave;
namespace Lilith.Core.Kokoro;
/// <summary>Local synthesis and Windows playback with tracked native ownership.</summary>
public sealed class KokoroTts : IAsyncDisposable
{
private readonly Logger _logger;
private readonly KokoroTtsOptions _options;
private readonly SemaphoreSlim _gate = new(1, 1);
private readonly AsyncResourceOwner _owner;
private KokoroWavSynthesizer? _model;
private KokoroVoice? _voice;
private static readonly object VoiceGate = new();
public KokoroTts(Logger logger, KokoroTtsOptions? options = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options ?? new();
_owner = new AsyncResourceOwner(() => { _model?.Dispose(); _model = null; _voice = null; _gate.Dispose(); },
TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
}
public string VoiceId => _voice?.Name ?? _options.VoiceId;
public bool WaitForTts => _options.WaitForTts;
public bool IsReady => !_owner.IsClosing && _model is not null && _voice is not null;
public static void ValidateVoice(string id) => _ = GetVoice(id);
private static KokoroVoice GetVoice(string id)
{
if (string.IsNullOrWhiteSpace(id)) throw new ArgumentException("Voice ID is required.", nameof(id));
lock (VoiceGate)
{
try { return KokoroVoiceManager.GetVoice(id); }
catch (InvalidOperationException ex) { throw new ArgumentException($"Unknown Kokoro voice '{id}'.", nameof(id), ex); }
}
}
public static async Task<KokoroTts> CreateAsync(Logger logger, KokoroTtsOptions? options = null, CancellationToken ct = default)
{
var service = new KokoroTts(logger, options);
try { await service.InitializeAsync(ct); return service; }
catch { await service.DisposeAsync(); throw; }
}
public Task InitializeAsync(CancellationToken ct = default) => _owner.RunAsync(async token =>
{
using var operation = await _gate.EnterAsync(token);
if (_model is not null) return;
var voice = GetVoice(_options.VoiceId);
string path = _options.ResolvedModelPath;
if (!File.Exists(path)) throw new FileNotFoundException("Run Kokoro setup to download the speech model.", path);
var model = await Task.Run(() => KokoroWavSynthesizer.LoadModel(path), CancellationToken.None);
if (token.IsCancellationRequested) { model.Dispose(); token.ThrowIfCancellationRequested(); }
_voice = voice;
_model = model;
Log("Kokoro ready (voice: " + VoiceId + ").", LogColors.Cyan);
}, ct);
public Task SetVoiceAsync(string id, CancellationToken ct = default) => _owner.RunAsync(async token =>
{
var voice = GetVoice(id);
using var operation = await _gate.EnterAsync(token);
_voice = voice;
}, ct);
public async Task SpeakAsync(string text, CancellationToken ct = default)
{
ct.ThrowIfCancellationRequested();
string spoken = KokoroTtsSanitizer.Sanitize(text);
if (spoken.Length == 0) return;
if (!IsReady) await InitializeAsync(ct);
Task monitored = ObserveSpeechAsync(_owner.RunAsync(token => SpeakCoreAsync(spoken, token), ct));
if (WaitForTts) await monitored;
}
/// <summary>Returns the actual synthesized 24 kHz, 16-bit mono PCM for recording/export.</summary>
public async Task<byte[]> SynthesizeAsync(string text, CancellationToken ct = default)
{
ct.ThrowIfCancellationRequested();
string spoken = KokoroTtsSanitizer.Sanitize(text);
if (spoken.Length == 0) return [];
if (!IsReady) await InitializeAsync(ct);
byte[] audio = [];
await _owner.RunAsync(async token =>
{
using var operation = await _gate.EnterAsync(token);
audio = await _model!.SynthesizeAsync(spoken, _voice!);
token.ThrowIfCancellationRequested();
if (audio.Length == 0) throw new InvalidDataException("Kokoro generated no audio.");
}, ct);
return audio;
}
/// <summary>Synthesizes once and aligns written words to the generated phoneme timestamps.</summary>
public async Task<KokoroNarration> SynthesizeTimedAsync(string text, CancellationToken ct = default)
{
string spoken = KokoroTtsSanitizer.Sanitize(text);
if (spoken.Length == 0) throw new InvalidDataException("Reply contains no speakable text.");
if (!IsReady) await InitializeAsync(ct);
KokoroNarration? narration = null;
await _owner.RunAsync(async token =>
{
using var operation = await _gate.EnterAsync(token);
var result = await _model!.SynthesizeWithTimestampsAsync(spoken, _voice!);
token.ThrowIfCancellationRequested();
if (result.AudioBytes.Length == 0) throw new InvalidDataException("Kokoro returned no speech audio.");
if (result.Timestamps.Length == 0)
{
narration = new(result.AudioBytes, EstimateWordTimes(spoken, result.AudioBytes), "estimated");
Log("This Kokoro model has no duration output; caption word timings are estimated.", LogColors.Orange);
return;
}
List<KokoroWord> words = [];
int previous = 0;
foreach (Match word in RegexPatterns.Words().Matches(spoken))
{
while (previous < result.Timestamps.Length - 1 && result.Timestamps[previous].Phoneme is ' ' or '$') previous++;
int first = Math.Clamp(previous, 0, result.Timestamps.Length - 1);
string phonemes = word.Value.Any(char.IsLetterOrDigit)
? Tokenizer.Phonemize(word.Value, _voice!.GetLangCode()) : word.Value;
int available = Math.Min(result.Timestamps.Length - first, phonemes.Length + 6);
int[] costs = [.. Enumerable.Range(0, available + 1)];
for (int row = 1; row <= phonemes.Length; row++)
{
int diagonal = costs[0]; costs[0] = row;
for (int column = 1; column <= available; column++)
{
int old = costs[column];
costs[column] = Math.Min(Math.Min(costs[column] + 1, costs[column - 1] + 1),
diagonal + (phonemes[row - 1] == result.Timestamps[first + column - 1].Phoneme ? 0 : 1));
diagonal = old;
}
}
int length = Enumerable.Range(1, available)
.OrderBy(n => costs[n] + Math.Abs(n - phonemes.Length) * .05).First();
int end = first + length, last = end - 1;
words.Add(new(word.Value, result.Timestamps[first].StartSecond, result.Timestamps[last].EndSecond));
previous = end;
}
narration = new(result.AudioBytes, words.ToArray());
}, ct);
return narration!;
}
private static KokoroWord[] EstimateWordTimes(string spoken, byte[] pcm)
{
// Map word-length weights onto voiced 10 ms windows, skipping actual silence.
List<double> windows = [];
const int samplesPerWindow = 240;
for (int offset = 0; offset < pcm.Length; offset += samplesPerWindow * 2)
{
double energy = 0; int count = 0;
for (int index = offset; index + 1 < Math.Min(pcm.Length, offset + samplesPerWindow * 2); index += 2)
{
double sample = (short)(pcm[index] | pcm[index + 1] << 8) / 32768.0;
energy += sample * sample; count++;
}
if (count > 0 && Math.Sqrt(energy / count) > .004) windows.Add(offset / 48000.0);
}
if (windows.Count == 0) throw new InvalidDataException("Kokoro speech audio is silent.");
var matches = RegexPatterns.Words().Matches(spoken).Cast<Match>().ToArray();
var weights = matches.Select(m => Math.Max(1, m.Value.Count(char.IsLetterOrDigit))).ToArray();
double total = weights.Sum(), used = 0;
List<KokoroWord> words = [];
for (int i = 0; i < matches.Length; i++)
{
int first = Math.Min(windows.Count - 1, (int)(used / total * windows.Count));
used += weights[i];
int last = Math.Min(windows.Count - 1, Math.Max(first, (int)(used / total * windows.Count) - 1));
words.Add(new(matches[i].Value, windows[first], windows[last] + .01));
}
return words.ToArray();
}
private async Task ObserveSpeechAsync(Task speech)
{
try { await speech; }
catch (OperationCanceledException) { Log("Speech canceled.", LogColors.Orange); }
catch (Exception ex) { Log($"Kokoro speech failed: {ex.Message}", LogColors.Red); }
}
private async Task SpeakCoreAsync(string text, CancellationToken token)
{
using var operation = await _gate.EnterAsync(token);
Progress("Generating speech...");
// Cancellation/deadlines bound the caller, but this work keeps ownership until native completion.
byte[] audio = await _model!.SynthesizeAsync(text, _voice!);
token.ThrowIfCancellationRequested();
if (audio.Length == 0) throw new InvalidDataException("Kokoro generated no audio.");
using var stream = new RawSourceWaveStream(audio, 0, audio.Length, new WaveFormat(24000, 16, 1));
using var player = new WaveOutEvent();
var complete = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
void Stopped(object? sender, StoppedEventArgs e)
{
if (e.Exception is not null) complete.TrySetException(e.Exception);
else complete.TrySetResult();
}
player.PlaybackStopped += Stopped;
try
{
player.Init(stream);
player.Play();
Progress("Speaking...");
await complete.Task.WaitAsync(TimeSpan.FromSeconds(audio.Length / 48000.0 + 10), token);
Log("Speech complete.");
}
finally { player.Stop(); player.PlaybackStopped -= Stopped; }
}
private void Log(string text, string color = LogColors.White)
{
_owner.WhileOpen(() => _logger.WriteLine("TTS", text, color));
}
private void Progress(string text)
{
_owner.WhileOpen(() => _logger.WriteProgress("TTS", text));
}
public async ValueTask DisposeAsync() => await _owner.DisposeAsync();
}