From 66ab678cb783f5f54bb2e62dc37d59042d7a6533 Mon Sep 17 00:00:00 2001 From: Shaw Date: Fri, 3 Jul 2026 00:28:36 -0400 Subject: [PATCH] feat(kokoro): IPA-input synthesis + G2P-kind query for espeak-less builds (#11776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fused eliza_inference_kokoro_synthesize() takes raw text and phonemizes inside the lib: kokoro_phonemize() uses real espeak-ng G2P only when KOKORO_USE_ESPEAK is compiled in, else falls back to a lossy per-byte ASCII grapheme map. Every build that does not link libespeak-ng (Android + iOS always; desktop whenever the host lacks libespeak-ng dev files) therefore produces speech-shaped but unintelligible audio. The kokoro lib already had eliza_kokoro::ipa_to_token_ids() (embedded 115-entry vocab, exact reference ids) — it just was not reachable through the fused FFI. Expose it additively (ABI v12 -> v14; v13 is the main-lineage vision surface, so this develop-pinned lineage advances to v14 to stay collision-free through the #11386 fork reconciliation): - eliza_inference_kokoro_g2p_kind(ctx): reports ELIZA_KOKORO_G2P_ESPEAK vs _ASCII so the caller knows whether it must pre-phonemize. - eliza_inference_kokoro_synthesize_ipa(ctx, ipa, ...): synthesize from precomputed espeak-ng IPA, routed through ipa_to_token_ids(), bypassing the in-lib phonemizer entirely. kokoro_synthesize() and the new kokoro_synthesize_ipa() now share one synthesis core (kokoro_synthesize_from_input_ids); the only difference is the G2P front end. All new symbols are additive — a v12/v13 caller is unaffected. Native tests extended: test_kokoro_phonemes asserts g2p_kind_of_build() mirrors espeak_available() and that the IPA-input path derives the exact wrapped input_ids; test_kokoro_g2p_espeak asserts g2p_kind == ESPEAK when linked. Refs elizaOS/eliza#11776, #10726, #10727, #11238. Co-Authored-By: Claude Fable 5 --- tools/kokoro/CMakeLists.txt | 11 ++- tools/kokoro/include/kokoro.h | 31 +++++- tools/kokoro/src/kokoro.cpp | 97 +++++++++++++++---- tools/kokoro/tests/test_kokoro_g2p_espeak.cpp | 16 +++ tools/kokoro/tests/test_kokoro_phonemes.cpp | 19 ++++ tools/omnivoice/include/eliza-inference-ffi.h | 55 ++++++++++- tools/omnivoice/src/eliza-inference-ffi.cpp | 62 ++++++++++++ 7 files changed, 262 insertions(+), 29 deletions(-) diff --git a/tools/kokoro/CMakeLists.txt b/tools/kokoro/CMakeLists.txt index 48998333f..cec127b2f 100644 --- a/tools/kokoro/CMakeLists.txt +++ b/tools/kokoro/CMakeLists.txt @@ -63,9 +63,12 @@ endif() # Real G2P via libespeak-ng. When present, kokoro_phonemize() drives # espeak_TextToPhonemes() (en-us IPA) and maps codepoints to Kokoro vocab ids, -# reproducing the reference token sequence. When absent, the build falls back -# to the degraded ASCII grapheme mapping and the TS layer must supply IPA. -# Override the search with -DKOKORO_ESPEAK_ROOT= (e.g. Homebrew). +# reproducing the reference token sequence. When absent, the text path falls +# back to the degraded ASCII grapheme mapping — so an espeak-less build reports +# `eliza_inference_kokoro_g2p_kind() == ELIZA_KOKORO_G2P_ASCII` and the TS voice +# layer supplies espeak-ng (WASM) IPA through +# `eliza_inference_kokoro_synthesize_ipa()` (ABI v14, #11776) to stay +# intelligible. Override the search with -DKOKORO_ESPEAK_ROOT= (Homebrew). option(KOKORO_ENABLE_ESPEAK "Link libespeak-ng for real Kokoro G2P" ON) if(KOKORO_ENABLE_ESPEAK) find_path(ESPEAK_NG_INCLUDE_DIR espeak-ng/speak_lib.h @@ -80,7 +83,7 @@ if(KOKORO_ENABLE_ESPEAK) target_compile_definitions(kokoro_lib PRIVATE KOKORO_USE_ESPEAK) message(STATUS "Kokoro G2P: libespeak-ng found (${ESPEAK_NG_LIBRARY}) — real IPA path enabled") else() - message(STATUS "Kokoro G2P: libespeak-ng not found — falling back to ASCII grapheme mapping (TS layer must supply IPA)") + message(STATUS "Kokoro G2P: libespeak-ng not found — ASCII grapheme fallback; g2p_kind() reports ASCII so the TS layer feeds IPA via kokoro_synthesize_ipa (#11776)") endif() endif() diff --git a/tools/kokoro/include/kokoro.h b/tools/kokoro/include/kokoro.h index b218677cf..ced649e89 100644 --- a/tools/kokoro/include/kokoro.h +++ b/tools/kokoro/include/kokoro.h @@ -121,12 +121,27 @@ kokoro_status kokoro_load_voice_preset( // real G2P path (text → en-us IPA → Kokoro vocab ids), reproducing the // upstream phonemizer's token sequence. Without libespeak-ng it falls back to // a deterministic (lossy) ASCII grapheme mapping; in that case the TS voice -// layer should phonemize and pass IPA (see kokoro-phonemes.h ipa_to_token_ids). +// layer should phonemize and pass IPA (see kokoro-phonemes.h ipa_to_token_ids) +// via `kokoro_synthesize_ipa()`. std::vector kokoro_phonemize(const std::string & text); +// Which grapheme→phoneme path this build uses, decided at compile time by +// whether libespeak-ng was linked (KOKORO_USE_ESPEAK). The caller queries this +// to decide whether it must supply its own IPA: an ESPEAK build phonemizes raw +// text correctly inside `kokoro_synthesize`, while an ASCII build only has the +// lossy grapheme fallback and needs `kokoro_synthesize_ipa` fed with real +// espeak-ng IPA from the TS layer. +enum kokoro_g2p_kind { + KOKORO_G2P_ASCII = 0, // no espeak — text path is lossy; use kokoro_synthesize_ipa + KOKORO_G2P_ESPEAK = 1, // real espeak-ng G2P — kokoro_synthesize(text) is correct +}; +kokoro_g2p_kind kokoro_g2p_kind_of_build() noexcept; + // Synthesize a single utterance. `text` is the natural-language input, // `voice` is the loaded ref_s preset. Output PCM lands in `out`. The // `speed_mult` parameter scales the predicted durations (1.0 = native rate). +// On an ASCII (espeak-less) build the internal grapheme fallback yields +// speech-shaped but unintelligible audio — use `kokoro_synthesize_ipa` there. kokoro_status kokoro_synthesize( const kokoro_model * model, const kokoro_voice_preset & voice, @@ -135,6 +150,20 @@ kokoro_status kokoro_synthesize( kokoro_audio & out, std::string & err_out) noexcept; +// Synthesize from a precomputed espeak-ng IPA string (UTF-8) instead of raw +// text. The IPA is mapped straight to Kokoro vocab ids via `ipa_to_token_ids` +// and wrapped as [PAD, *ids, PAD] — bypassing `kokoro_phonemize` entirely, so +// this is the intelligible path on ASCII (espeak-less) builds where the TS +// layer runs its own espeak-ng (WASM) phonemizer and passes the IPA in. +// Identical synthesis contract to `kokoro_synthesize` otherwise. +kokoro_status kokoro_synthesize_ipa( + const kokoro_model * model, + const kokoro_voice_preset & voice, + const std::string & ipa, + float speed_mult, + kokoro_audio & out, + std::string & err_out) noexcept; + // Returns the model's audio sample rate (24000 for v1.0). int kokoro_sample_rate(const kokoro_model * model) noexcept; diff --git a/tools/kokoro/src/kokoro.cpp b/tools/kokoro/src/kokoro.cpp index b03fa2ff7..a81d173d4 100644 --- a/tools/kokoro/src/kokoro.cpp +++ b/tools/kokoro/src/kokoro.cpp @@ -480,6 +480,10 @@ std::vector kokoro_phonemize(const std::string & text) { return phonemize_ascii(text); } +kokoro_g2p_kind kokoro_g2p_kind_of_build() noexcept { + return espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII; +} + // --------------------------------------------------------------------------- // Synthesis path // --------------------------------------------------------------------------- @@ -501,36 +505,23 @@ std::vector kokoro_phonemize(const std::string & text) { // for the next training/inference wave. -kokoro_status kokoro_synthesize( +// Shared synthesis core: takes an already-phonemized, wrapped input-id run +// ([PAD, *ids, PAD]) and runs the predictor → decoder → 24 kHz PCM path. Both +// public entries (`kokoro_synthesize` = raw text via kokoro_phonemize, +// `kokoro_synthesize_ipa` = precomputed espeak IPA via ipa_to_token_ids) funnel +// here so the two G2P front-ends share one identical back-end. `model`/`voice` +// are already validated by the caller; this takes the model lock. +static kokoro_status kokoro_synthesize_from_input_ids( const kokoro_model * model, const kokoro_voice_preset & voice, - const std::string & text, + std::vector phonemes, float speed_mult, kokoro_audio & out, std::string & err_out) noexcept { - err_out.clear(); - out.samples.clear(); - out.sample_rate = 24000; - - if (!model) { - err_out = "null model"; - return KOKORO_E_INVALID_ARG; - } - if (voice.data.empty() || voice.style_dim <= 0) { - err_out = "empty / malformed voice preset"; - return KOKORO_E_INVALID_ARG; - } - if (text.empty()) { - err_out = "empty text"; - return KOKORO_E_INVALID_ARG; - } - std::lock_guard lk(const_cast(model)->mu); out.sample_rate = model->hparams.sample_rate; - // 1. Phonemize. - std::vector phonemes = kokoro_phonemize(text); if (phonemes.size() > 510) phonemes.resize(510); // 2. Slice ref_s — kokoro-onnx uses voice[len(tokens)] where `tokens` is @@ -626,6 +617,70 @@ kokoro_status kokoro_synthesize( } +// Shared model/voice validation for both public synthesis entries. +static kokoro_status kokoro_validate_synth_args( + const kokoro_model * model, + const kokoro_voice_preset & voice, + std::string & err_out) noexcept { + if (!model) { + err_out = "null model"; + return KOKORO_E_INVALID_ARG; + } + if (voice.data.empty() || voice.style_dim <= 0) { + err_out = "empty / malformed voice preset"; + return KOKORO_E_INVALID_ARG; + } + return KOKORO_OK; +} + +kokoro_status kokoro_synthesize( + const kokoro_model * model, + const kokoro_voice_preset & voice, + const std::string & text, + float speed_mult, + kokoro_audio & out, + std::string & err_out) noexcept { + err_out.clear(); + out.samples.clear(); + out.sample_rate = 24000; + + kokoro_status vst = kokoro_validate_synth_args(model, voice, err_out); + if (vst != KOKORO_OK) return vst; + if (text.empty()) { + err_out = "empty text"; + return KOKORO_E_INVALID_ARG; + } + + // 1. Phonemize (espeak-ng IPA when linked, else lossy ASCII grapheme map). + return kokoro_synthesize_from_input_ids( + model, voice, kokoro_phonemize(text), speed_mult, out, err_out); +} + +kokoro_status kokoro_synthesize_ipa( + const kokoro_model * model, + const kokoro_voice_preset & voice, + const std::string & ipa, + float speed_mult, + kokoro_audio & out, + std::string & err_out) noexcept { + err_out.clear(); + out.samples.clear(); + out.sample_rate = 24000; + + kokoro_status vst = kokoro_validate_synth_args(model, voice, err_out); + if (vst != KOKORO_OK) return vst; + if (ipa.empty()) { + err_out = "empty ipa"; + return KOKORO_E_INVALID_ARG; + } + + // 1. Map the caller-supplied espeak-ng IPA straight to Kokoro vocab ids, + // wrapped as the model input_ids [PAD, *ids, PAD] — no internal G2P. + return kokoro_synthesize_from_input_ids( + model, voice, wrap_input_ids(ipa_to_token_ids(ipa)), speed_mult, out, + err_out); +} + int kokoro_sample_rate(const kokoro_model * model) noexcept { return model ? model->hparams.sample_rate : 24000; } diff --git a/tools/kokoro/tests/test_kokoro_g2p_espeak.cpp b/tools/kokoro/tests/test_kokoro_g2p_espeak.cpp index 75eea6dfb..0e9c2ab03 100644 --- a/tools/kokoro/tests/test_kokoro_g2p_espeak.cpp +++ b/tools/kokoro/tests/test_kokoro_g2p_espeak.cpp @@ -6,6 +6,7 @@ // -L /opt/homebrew/lib -lespeak-ng -o /tmp/t && /tmp/t #include "kokoro-phonemes.h" +#include "kokoro.h" #include #include @@ -31,6 +32,21 @@ int main() { int fails = 0; + // G2P-kind capability (ABI v14, #11776): a build that links libespeak-ng + // must report ESPEAK so the TS layer keeps sending raw text; without it the + // build reports ASCII and the TS layer feeds IPA via kokoro_synthesize_ipa. + { + const kokoro_g2p_kind kind = kokoro_g2p_kind_of_build(); + const kokoro_g2p_kind want = + espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII; + const bool ok = kind == want; + printf("g2p_kind_of_build: %s (want %s): %s\n", + kind == KOKORO_G2P_ESPEAK ? "espeak" : "ascii", + want == KOKORO_G2P_ESPEAK ? "espeak" : "ascii", + ok ? "PASS" : "FAIL"); + if (!ok) ++fails; + } + // Reference case (from reference-ids.json). { const std::string text = "Hello, this is a native Kokoro voice test."; diff --git a/tools/kokoro/tests/test_kokoro_phonemes.cpp b/tools/kokoro/tests/test_kokoro_phonemes.cpp index 6df30956a..83b3312c6 100644 --- a/tools/kokoro/tests/test_kokoro_phonemes.cpp +++ b/tools/kokoro/tests/test_kokoro_phonemes.cpp @@ -9,6 +9,7 @@ // dependency-free. #include "kokoro-phonemes.h" +#include "kokoro.h" #include #include @@ -74,6 +75,24 @@ int main() { const std::vector exp = {KOKORO_PAD_ID, 50, 51, KOKORO_PAD_ID}; assert(ids == exp); } + { + // G2P-kind capability query mirrors espeak_available() (ABI v14, #11776). + // A build without libespeak-ng must report ASCII so the TS layer knows it + // has to feed IPA through kokoro_synthesize_ipa. + const kokoro_g2p_kind kind = kokoro_g2p_kind_of_build(); + assert(kind == (espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII)); + } + { + // The IPA-input synthesis path derives its model input_ids as + // wrap_input_ids(ipa_to_token_ids(ipa)) — assert the exact wrapped run + // the native kokoro_synthesize_ipa entry would feed the model for + // "həlˈoʊ". + const std::string ipa = "h\xC9\x99l\xCB\x88o\xCA\x8A"; + std::vector input = wrap_input_ids(ipa_to_token_ids(ipa)); + const std::vector exp = { + KOKORO_PAD_ID, 50, 83, 54, 156, 57, 135, KOKORO_PAD_ID}; + assert(input == exp); + } std::printf("test_kokoro_phonemes: OK\n"); return 0; diff --git a/tools/omnivoice/include/eliza-inference-ffi.h b/tools/omnivoice/include/eliza-inference-ffi.h index 492626a99..8ee29bbaa 100644 --- a/tools/omnivoice/include/eliza-inference-ffi.h +++ b/tools/omnivoice/include/eliza-inference-ffi.h @@ -1,5 +1,5 @@ /* - * libelizainference FFI ABI v12. + * libelizainference FFI ABI v14. * * (Banner tracks ELIZA_INFERENCE_ABI_VERSION below; the per-version history is * at the end of this header preamble, newest first.) @@ -134,6 +134,24 @@ extern "C" { * load and refuses to bind if they disagree. * * Changelog: + * v14: Kokoro IPA input + G2P-kind capability query. + * `eliza_inference_kokoro_g2p_kind()` reports whether the linked + * kokoro_lib phonemizes raw text with real espeak-ng + * (ELIZA_KOKORO_G2P_ESPEAK) or only the lossy per-byte ASCII fallback + * (ELIZA_KOKORO_G2P_ASCII). `eliza_inference_kokoro_synthesize_ipa()` + * takes precomputed espeak-ng IPA and routes it through the fixed Kokoro + * vocab (`ipa_to_token_ids`), bypassing the in-lib phonemizer — the + * intelligible path for espeak-less Android / iOS / host builds, fed by + * the TS espeak-ng-WASM phonemizer (#11776). Before this, an espeak-less + * fused build phonemized raw text with the ASCII grapheme fallback and + * produced speech-shaped but unintelligible audio on every mobile build + * and any host without libespeak-ng. Additive symbols — a v12/v13 caller + * is unaffected; a library that predates this surface reports the symbols + * absent and the loader falls back to the raw-text path. + * NOTE ON NUMBERING: v13 (token-by-token vision describe) is the + * main-lineage vision surface. This develop-pinned lineage (fork-sync + * #11386) advances 12 -> 14 for the Kokoro IPA surface so the two + * independent bumps stay collision-free through reconciliation. * v12: ASR word timestamps folded into the fused ASR. * `eliza_inference_asr_timestamps_supported()` + `_asr_transcribe_timed` * run the SAME audio-in/text-out decode as `_asr_transcribe` and @@ -203,9 +221,9 @@ extern "C" { * v7: real Silero VAD (same symbol surface as v6). * v6: fused wake-word, speaker, diarizer. */ -#define ELIZA_INFERENCE_ABI_VERSION 12 +#define ELIZA_INFERENCE_ABI_VERSION 14 -/* Returns a static, NUL-terminated string of the form "12" matching +/* Returns a static, NUL-terminated string of the form "14" matching * ELIZA_INFERENCE_ABI_VERSION at the time the library was built. The * pointer is owned by the library — do NOT free. */ const char * eliza_inference_abi_version(void); @@ -391,6 +409,37 @@ int eliza_inference_kokoro_synthesize( * negative ELIZA_* code if no Kokoro model is loaded. */ int eliza_inference_kokoro_sample_rate(EliInferenceContext * ctx); +/* ---- Kokoro G2P kind + IPA synthesis (ABI v14) -------------------- * + * + * Which grapheme->phoneme path the linked kokoro_lib uses, so the TS voice + * layer can decide whether it must pre-phonemize: + * ELIZA_KOKORO_G2P_ESPEAK (1): the lib links libespeak-ng and phonemizes raw + * text correctly inside `eliza_inference_kokoro_synthesize` — pass text. + * ELIZA_KOKORO_G2P_ASCII (0): the lib has NO espeak; the text path uses the + * lossy per-byte ASCII grapheme fallback (unintelligible). The caller MUST + * run its own espeak-ng G2P and call + * `eliza_inference_kokoro_synthesize_ipa` with the IPA instead. + * This is a build property (independent of any loaded model); `ctx` is accepted + * for API symmetry. Returns a negative ELIZA_* code on a non-Kokoro build. */ +#define ELIZA_KOKORO_G2P_ASCII 0 +#define ELIZA_KOKORO_G2P_ESPEAK 1 +int eliza_inference_kokoro_g2p_kind(EliInferenceContext * ctx); + +/* Synthesize from precomputed espeak-ng IPA (UTF-8) instead of raw text: the + * IPA is mapped straight to Kokoro vocab ids (`ipa_to_token_ids`), bypassing the + * in-lib phonemizer. This is the intelligible path on espeak-less builds + * (Android / iOS / host-without-libespeak) — the TS layer runs the espeak-ng + * WASM phonemizer and passes the IPA here. Same return contract as + * `eliza_inference_kokoro_synthesize`. */ +int eliza_inference_kokoro_synthesize_ipa( + EliInferenceContext * ctx, + const char * ipa, + size_t ipa_len, + float speed, + float * out_pcm, + size_t max_samples, + char ** out_error); + /* ---- OmniVoice reference encode (ABI v4) -------------------------- * * * Encode a 24 kHz mono fp32 PCM buffer through the OmniVoice tokenizer diff --git a/tools/omnivoice/src/eliza-inference-ffi.cpp b/tools/omnivoice/src/eliza-inference-ffi.cpp index a0390cacc..a50ddf79c 100644 --- a/tools/omnivoice/src/eliza-inference-ffi.cpp +++ b/tools/omnivoice/src/eliza-inference-ffi.cpp @@ -1824,6 +1824,68 @@ int eliza_inference_kokoro_sample_rate(EliInferenceContext * ctx) { #endif } +int eliza_inference_kokoro_g2p_kind(EliInferenceContext * ctx) { +#ifdef ELIZA_ENABLE_KOKORO + /* Build property — independent of any loaded model. ctx accepted for API + * symmetry only. ELIZA_KOKORO_G2P_{ESPEAK,ASCII} mirror the kokoro_lib enum + * so a numeric-equal contract holds across the header boundary. */ + (void) ctx; + return eliza_kokoro::kokoro_g2p_kind_of_build() == eliza_kokoro::KOKORO_G2P_ESPEAK + ? ELIZA_KOKORO_G2P_ESPEAK + : ELIZA_KOKORO_G2P_ASCII; +#else + (void) ctx; + return ELIZA_ERR_NOT_IMPLEMENTED; +#endif +} + +int eliza_inference_kokoro_synthesize_ipa( + EliInferenceContext * ctx, + const char * ipa, + size_t ipa_len, + float speed, + float * out_pcm, + size_t max_samples, + char ** out_error) { +#ifdef ELIZA_ENABLE_KOKORO + if (!ctx || !ipa || !out_pcm) { + eliza_set_error(out_error, "[libelizainference] kokoro_synthesize_ipa: null argument"); + return ELIZA_ERR_INVALID_ARG; + } + std::lock_guard lock(ctx->kokoro_mutex); + if (!ctx->kokoro_loaded || !ctx->kokoro_model) { + eliza_set_error(out_error, + "[libelizainference] kokoro_synthesize_ipa: no model loaded (call kokoro_load first)"); + return ELIZA_ERR_INVALID_ARG; + } + if (speed <= 0.0f) speed = 1.0f; + const std::string in(ipa, ipa_len ? ipa_len : std::strlen(ipa)); + eliza_kokoro::kokoro_audio audio; + std::string err; + eliza_kokoro::kokoro_status st = eliza_kokoro::kokoro_synthesize_ipa( + ctx->kokoro_model.get(), ctx->kokoro_voice, in, speed, audio, err); + if (st != eliza_kokoro::KOKORO_OK) { + eliza_set_error(out_error, + std::string("[libelizainference] kokoro_synthesize_ipa failed: ") + err); + return ELIZA_ERR_FFI_FAULT; + } + if (audio.samples.size() > max_samples) { + eliza_set_error(out_error, + "[libelizainference] kokoro_synthesize_ipa: out_pcm too small; required samples=" + + std::to_string(audio.samples.size())); + return ELIZA_ERR_INVALID_ARG; + } + std::memcpy(out_pcm, audio.samples.data(), audio.samples.size() * sizeof(float)); + return (int) audio.samples.size(); +#else + (void) ctx; (void) ipa; (void) ipa_len; (void) speed; + (void) out_pcm; (void) max_samples; + eliza_set_error(out_error, + "[libelizainference] kokoro not built (ELIZA_ENABLE_KOKORO off)"); + return ELIZA_ERR_NOT_IMPLEMENTED; +#endif +} + int eliza_inference_mmap_acquire( EliInferenceContext * ctx, const char * region_name,