feat(kokoro): IPA-input synthesis + G2P-kind query for espeak-less builds (#11776)

tools/kokoro/CMakeLists.txt

@@ -63,9 +63,12 @@ endif()
 
 # Real G2P via libespeak-ng. When present, kokoro_phonemize() drives
 # espeak_TextToPhonemes() (en-us IPA) and maps codepoints to Kokoro vocab ids,
-# reproducing the reference token sequence. When absent, the build falls back
-# to the degraded ASCII grapheme mapping and the TS layer must supply IPA.
-# Override the search with -DKOKORO_ESPEAK_ROOT=<prefix> (e.g. Homebrew).
+# reproducing the reference token sequence. When absent, the text path falls
+# back to the degraded ASCII grapheme mapping — so an espeak-less build reports
+# `eliza_inference_kokoro_g2p_kind() == ELIZA_KOKORO_G2P_ASCII` and the TS voice
+# layer supplies espeak-ng (WASM) IPA through
+# `eliza_inference_kokoro_synthesize_ipa()` (ABI v14, #11776) to stay
+# intelligible. Override the search with -DKOKORO_ESPEAK_ROOT=<prefix> (Homebrew).
 option(KOKORO_ENABLE_ESPEAK "Link libespeak-ng for real Kokoro G2P" ON)
 if(KOKORO_ENABLE_ESPEAK)
     find_path(ESPEAK_NG_INCLUDE_DIR espeak-ng/speak_lib.h
@@ -80,7 +83,7 @@ if(KOKORO_ENABLE_ESPEAK)
         target_compile_definitions(kokoro_lib PRIVATE KOKORO_USE_ESPEAK)
         message(STATUS "Kokoro G2P: libespeak-ng found (${ESPEAK_NG_LIBRARY}) — real IPA path enabled")
     else()
-        message(STATUS "Kokoro G2P: libespeak-ng not found — falling back to ASCII grapheme mapping (TS layer must supply IPA)")
+        message(STATUS "Kokoro G2P: libespeak-ng not found — ASCII grapheme fallback; g2p_kind() reports ASCII so the TS layer feeds IPA via kokoro_synthesize_ipa (#11776)")
     endif()
 endif()
 

tools/kokoro/include/kokoro.h

@@ -121,12 +121,27 @@ kokoro_status kokoro_load_voice_preset(
 // real G2P path (text → en-us IPA → Kokoro vocab ids), reproducing the
 // upstream phonemizer's token sequence. Without libespeak-ng it falls back to
 // a deterministic (lossy) ASCII grapheme mapping; in that case the TS voice
-// layer should phonemize and pass IPA (see kokoro-phonemes.h ipa_to_token_ids).
+// layer should phonemize and pass IPA (see kokoro-phonemes.h ipa_to_token_ids)
+// via `kokoro_synthesize_ipa()`.
 std::vector<int32_t> kokoro_phonemize(const std::string & text);
 
+// Which grapheme→phoneme path this build uses, decided at compile time by
+// whether libespeak-ng was linked (KOKORO_USE_ESPEAK). The caller queries this
+// to decide whether it must supply its own IPA: an ESPEAK build phonemizes raw
+// text correctly inside `kokoro_synthesize`, while an ASCII build only has the
+// lossy grapheme fallback and needs `kokoro_synthesize_ipa` fed with real
+// espeak-ng IPA from the TS layer.
+enum kokoro_g2p_kind {
+    KOKORO_G2P_ASCII  = 0,  // no espeak — text path is lossy; use kokoro_synthesize_ipa
+    KOKORO_G2P_ESPEAK = 1,  // real espeak-ng G2P — kokoro_synthesize(text) is correct
+};
+kokoro_g2p_kind kokoro_g2p_kind_of_build() noexcept;
+
 // Synthesize a single utterance. `text` is the natural-language input,
 // `voice` is the loaded ref_s preset. Output PCM lands in `out`. The
 // `speed_mult` parameter scales the predicted durations (1.0 = native rate).
+// On an ASCII (espeak-less) build the internal grapheme fallback yields
+// speech-shaped but unintelligible audio — use `kokoro_synthesize_ipa` there.
 kokoro_status kokoro_synthesize(
     const kokoro_model * model,
     const kokoro_voice_preset & voice,
@@ -135,6 +150,20 @@ kokoro_status kokoro_synthesize(
     kokoro_audio & out,
     std::string & err_out) noexcept;
 
+// Synthesize from a precomputed espeak-ng IPA string (UTF-8) instead of raw
+// text. The IPA is mapped straight to Kokoro vocab ids via `ipa_to_token_ids`
+// and wrapped as [PAD, *ids, PAD] — bypassing `kokoro_phonemize` entirely, so
+// this is the intelligible path on ASCII (espeak-less) builds where the TS
+// layer runs its own espeak-ng (WASM) phonemizer and passes the IPA in.
+// Identical synthesis contract to `kokoro_synthesize` otherwise.
+kokoro_status kokoro_synthesize_ipa(
+    const kokoro_model * model,
+    const kokoro_voice_preset & voice,
+    const std::string & ipa,
+    float speed_mult,
+    kokoro_audio & out,
+    std::string & err_out) noexcept;
+
 // Returns the model's audio sample rate (24000 for v1.0).
 int kokoro_sample_rate(const kokoro_model * model) noexcept;
 

tools/kokoro/src/kokoro.cpp

@@ -480,6 +480,10 @@ std::vector<int32_t> kokoro_phonemize(const std::string & text) {
     return phonemize_ascii(text);
 }
 
+kokoro_g2p_kind kokoro_g2p_kind_of_build() noexcept {
+    return espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII;
+}
+
 // ---------------------------------------------------------------------------
 // Synthesis path
 // ---------------------------------------------------------------------------
@@ -501,36 +505,23 @@ std::vector<int32_t> kokoro_phonemize(const std::string & text) {
 // for the next training/inference wave.
 
 
-kokoro_status kokoro_synthesize(
+// Shared synthesis core: takes an already-phonemized, wrapped input-id run
+// ([PAD, *ids, PAD]) and runs the predictor → decoder → 24 kHz PCM path. Both
+// public entries (`kokoro_synthesize` = raw text via kokoro_phonemize,
+// `kokoro_synthesize_ipa` = precomputed espeak IPA via ipa_to_token_ids) funnel
+// here so the two G2P front-ends share one identical back-end. `model`/`voice`
+// are already validated by the caller; this takes the model lock.
+static kokoro_status kokoro_synthesize_from_input_ids(
         const kokoro_model * model,
         const kokoro_voice_preset & voice,
-        const std::string & text,
+        std::vector<int32_t> phonemes,
         float speed_mult,
         kokoro_audio & out,
         std::string & err_out) noexcept {
-    err_out.clear();
-    out.samples.clear();
-    out.sample_rate = 24000;
-
-    if (!model) {
-        err_out = "null model";
-        return KOKORO_E_INVALID_ARG;
-    }
-    if (voice.data.empty() || voice.style_dim <= 0) {
-        err_out = "empty / malformed voice preset";
-        return KOKORO_E_INVALID_ARG;
-    }
-    if (text.empty()) {
-        err_out = "empty text";
-        return KOKORO_E_INVALID_ARG;
-    }
-
     std::lock_guard<std::mutex> lk(const_cast<kokoro_model *>(model)->mu);
 
     out.sample_rate = model->hparams.sample_rate;
 
-    // 1. Phonemize.
-    std::vector<int32_t> phonemes = kokoro_phonemize(text);
     if (phonemes.size() > 510) phonemes.resize(510);
 
     // 2. Slice ref_s — kokoro-onnx uses voice[len(tokens)] where `tokens` is
@@ -626,6 +617,70 @@ kokoro_status kokoro_synthesize(
 
 }
 
+// Shared model/voice validation for both public synthesis entries.
+static kokoro_status kokoro_validate_synth_args(
+        const kokoro_model * model,
+        const kokoro_voice_preset & voice,
+        std::string & err_out) noexcept {
+    if (!model) {
+        err_out = "null model";
+        return KOKORO_E_INVALID_ARG;
+    }
+    if (voice.data.empty() || voice.style_dim <= 0) {
+        err_out = "empty / malformed voice preset";
+        return KOKORO_E_INVALID_ARG;
+    }
+    return KOKORO_OK;
+}
+
+kokoro_status kokoro_synthesize(
+        const kokoro_model * model,
+        const kokoro_voice_preset & voice,
+        const std::string & text,
+        float speed_mult,
+        kokoro_audio & out,
+        std::string & err_out) noexcept {
+    err_out.clear();
+    out.samples.clear();
+    out.sample_rate = 24000;
+
+    kokoro_status vst = kokoro_validate_synth_args(model, voice, err_out);
+    if (vst != KOKORO_OK) return vst;
+    if (text.empty()) {
+        err_out = "empty text";
+        return KOKORO_E_INVALID_ARG;
+    }
+
+    // 1. Phonemize (espeak-ng IPA when linked, else lossy ASCII grapheme map).
+    return kokoro_synthesize_from_input_ids(
+        model, voice, kokoro_phonemize(text), speed_mult, out, err_out);
+}
+
+kokoro_status kokoro_synthesize_ipa(
+        const kokoro_model * model,
+        const kokoro_voice_preset & voice,
+        const std::string & ipa,
+        float speed_mult,
+        kokoro_audio & out,
+        std::string & err_out) noexcept {
+    err_out.clear();
+    out.samples.clear();
+    out.sample_rate = 24000;
+
+    kokoro_status vst = kokoro_validate_synth_args(model, voice, err_out);
+    if (vst != KOKORO_OK) return vst;
+    if (ipa.empty()) {
+        err_out = "empty ipa";
+        return KOKORO_E_INVALID_ARG;
+    }
+
+    // 1. Map the caller-supplied espeak-ng IPA straight to Kokoro vocab ids,
+    //    wrapped as the model input_ids [PAD, *ids, PAD] — no internal G2P.
+    return kokoro_synthesize_from_input_ids(
+        model, voice, wrap_input_ids(ipa_to_token_ids(ipa)), speed_mult, out,
+        err_out);
+}
+
 int kokoro_sample_rate(const kokoro_model * model) noexcept {
     return model ? model->hparams.sample_rate : 24000;
 }

tools/kokoro/tests/test_kokoro_g2p_espeak.cpp

@@ -6,6 +6,7 @@
 //     -L /opt/homebrew/lib -lespeak-ng -o /tmp/t && /tmp/t
 
 #include "kokoro-phonemes.h"
+#include "kokoro.h"
 
 #include <cstdio>
 #include <string>
@@ -31,6 +32,21 @@ int main() {
 
     int fails = 0;
 
+    // G2P-kind capability (ABI v14, #11776): a build that links libespeak-ng
+    // must report ESPEAK so the TS layer keeps sending raw text; without it the
+    // build reports ASCII and the TS layer feeds IPA via kokoro_synthesize_ipa.
+    {
+        const kokoro_g2p_kind kind = kokoro_g2p_kind_of_build();
+        const kokoro_g2p_kind want =
+            espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII;
+        const bool ok = kind == want;
+        printf("g2p_kind_of_build: %s (want %s): %s\n",
+               kind == KOKORO_G2P_ESPEAK ? "espeak" : "ascii",
+               want == KOKORO_G2P_ESPEAK ? "espeak" : "ascii",
+               ok ? "PASS" : "FAIL");
+        if (!ok) ++fails;
+    }
+
     // Reference case (from reference-ids.json).
     {
         const std::string text = "Hello, this is a native Kokoro voice test.";

tools/kokoro/tests/test_kokoro_phonemes.cpp

@@ -9,6 +9,7 @@
 // dependency-free.
 
 #include "kokoro-phonemes.h"
+#include "kokoro.h"
 
 #include <cassert>
 #include <cstdio>
@@ -74,6 +75,24 @@ int main() {
         const std::vector<int32_t> exp = {KOKORO_PAD_ID, 50, 51, KOKORO_PAD_ID};
         assert(ids == exp);
     }
+    {
+        // G2P-kind capability query mirrors espeak_available() (ABI v14, #11776).
+        // A build without libespeak-ng must report ASCII so the TS layer knows it
+        // has to feed IPA through kokoro_synthesize_ipa.
+        const kokoro_g2p_kind kind = kokoro_g2p_kind_of_build();
+        assert(kind == (espeak_available() ? KOKORO_G2P_ESPEAK : KOKORO_G2P_ASCII));
+    }
+    {
+        // The IPA-input synthesis path derives its model input_ids as
+        // wrap_input_ids(ipa_to_token_ids(ipa)) — assert the exact wrapped run
+        // the native kokoro_synthesize_ipa entry would feed the model for
+        // "həlˈoʊ".
+        const std::string ipa = "h\xC9\x99l\xCB\x88o\xCA\x8A";
+        std::vector<int32_t> input = wrap_input_ids(ipa_to_token_ids(ipa));
+        const std::vector<int32_t> exp = {
+            KOKORO_PAD_ID, 50, 83, 54, 156, 57, 135, KOKORO_PAD_ID};
+        assert(input == exp);
+    }
 
     std::printf("test_kokoro_phonemes: OK\n");
     return 0;

tools/omnivoice/include/eliza-inference-ffi.h

@@ -1,5 +1,5 @@
 /*
- * libelizainference FFI ABI v12.
+ * libelizainference FFI ABI v14.
  *
  * (Banner tracks ELIZA_INFERENCE_ABI_VERSION below; the per-version history is
  * at the end of this header preamble, newest first.)
@@ -134,6 +134,24 @@ extern "C" {
  * load and refuses to bind if they disagree.
  *
  * Changelog:
+ *   v14: Kokoro IPA input + G2P-kind capability query.
+ *        `eliza_inference_kokoro_g2p_kind()` reports whether the linked
+ *        kokoro_lib phonemizes raw text with real espeak-ng
+ *        (ELIZA_KOKORO_G2P_ESPEAK) or only the lossy per-byte ASCII fallback
+ *        (ELIZA_KOKORO_G2P_ASCII). `eliza_inference_kokoro_synthesize_ipa()`
+ *        takes precomputed espeak-ng IPA and routes it through the fixed Kokoro
+ *        vocab (`ipa_to_token_ids`), bypassing the in-lib phonemizer — the
+ *        intelligible path for espeak-less Android / iOS / host builds, fed by
+ *        the TS espeak-ng-WASM phonemizer (#11776). Before this, an espeak-less
+ *        fused build phonemized raw text with the ASCII grapheme fallback and
+ *        produced speech-shaped but unintelligible audio on every mobile build
+ *        and any host without libespeak-ng. Additive symbols — a v12/v13 caller
+ *        is unaffected; a library that predates this surface reports the symbols
+ *        absent and the loader falls back to the raw-text path.
+ *        NOTE ON NUMBERING: v13 (token-by-token vision describe) is the
+ *        main-lineage vision surface. This develop-pinned lineage (fork-sync
+ *        #11386) advances 12 -> 14 for the Kokoro IPA surface so the two
+ *        independent bumps stay collision-free through reconciliation.
  *   v12: ASR word timestamps folded into the fused ASR.
  *        `eliza_inference_asr_timestamps_supported()` + `_asr_transcribe_timed`
  *        run the SAME audio-in/text-out decode as `_asr_transcribe` and
@@ -203,9 +221,9 @@ extern "C" {
  *   v7: real Silero VAD (same symbol surface as v6).
  *   v6: fused wake-word, speaker, diarizer.
  */
-#define ELIZA_INFERENCE_ABI_VERSION 12
+#define ELIZA_INFERENCE_ABI_VERSION 14
 
-/* Returns a static, NUL-terminated string of the form "12" matching
+/* Returns a static, NUL-terminated string of the form "14" matching
  * ELIZA_INFERENCE_ABI_VERSION at the time the library was built. The
  * pointer is owned by the library — do NOT free. */
 const char * eliza_inference_abi_version(void);
@@ -391,6 +409,37 @@ int eliza_inference_kokoro_synthesize(
  * negative ELIZA_* code if no Kokoro model is loaded. */
 int eliza_inference_kokoro_sample_rate(EliInferenceContext * ctx);
 
+/* ---- Kokoro G2P kind + IPA synthesis (ABI v14) -------------------- *
+ *
+ * Which grapheme->phoneme path the linked kokoro_lib uses, so the TS voice
+ * layer can decide whether it must pre-phonemize:
+ *   ELIZA_KOKORO_G2P_ESPEAK (1): the lib links libespeak-ng and phonemizes raw
+ *       text correctly inside `eliza_inference_kokoro_synthesize` — pass text.
+ *   ELIZA_KOKORO_G2P_ASCII (0): the lib has NO espeak; the text path uses the
+ *       lossy per-byte ASCII grapheme fallback (unintelligible). The caller MUST
+ *       run its own espeak-ng G2P and call
+ *       `eliza_inference_kokoro_synthesize_ipa` with the IPA instead.
+ * This is a build property (independent of any loaded model); `ctx` is accepted
+ * for API symmetry. Returns a negative ELIZA_* code on a non-Kokoro build. */
+#define ELIZA_KOKORO_G2P_ASCII  0
+#define ELIZA_KOKORO_G2P_ESPEAK 1
+int eliza_inference_kokoro_g2p_kind(EliInferenceContext * ctx);
+
+/* Synthesize from precomputed espeak-ng IPA (UTF-8) instead of raw text: the
+ * IPA is mapped straight to Kokoro vocab ids (`ipa_to_token_ids`), bypassing the
+ * in-lib phonemizer. This is the intelligible path on espeak-less builds
+ * (Android / iOS / host-without-libespeak) — the TS layer runs the espeak-ng
+ * WASM phonemizer and passes the IPA here. Same return contract as
+ * `eliza_inference_kokoro_synthesize`. */
+int eliza_inference_kokoro_synthesize_ipa(
+    EliInferenceContext * ctx,
+    const char * ipa,
+    size_t ipa_len,
+    float speed,
+    float * out_pcm,
+    size_t max_samples,
+    char ** out_error);
+
 /* ---- OmniVoice reference encode (ABI v4) -------------------------- *
  *
  * Encode a 24 kHz mono fp32 PCM buffer through the OmniVoice tokenizer