From 10160b5eff6327b24422b801579752e0bade128b Mon Sep 17 00:00:00 2001 From: Hyeseong Kim Date: Wed, 22 Jul 2026 16:05:45 +0900 Subject: [PATCH 1/4] revert grapheme-counter.js separated entry --- .changeset/grapheme-counter-entry.md | 11 - .../bundle-entries/unicode-segmenter-both.js | 10 - .../unicode-segmenter-counter.js | 3 - .../bundle-entries/unicode-segmenter-full.js | 1 + .../bundle-entries/unicode-segmenter.js | 2 - benchmark/grapheme/bundle-stats-hermes.js | 3 +- benchmark/grapheme/bundle-stats.js | 3 +- benchmark/grapheme/memory-stats.js | 2 +- package.json | 5 - src/grapheme-core.js | 199 ------------------ src/grapheme-counter.js | 56 ----- src/grapheme.js | 198 ++++++++++++++++- test/corpora.js | 4 - test/grapheme.js | 14 +- 14 files changed, 200 insertions(+), 311 deletions(-) delete mode 100644 .changeset/grapheme-counter-entry.md delete mode 100644 benchmark/grapheme/bundle-entries/unicode-segmenter-both.js delete mode 100644 benchmark/grapheme/bundle-entries/unicode-segmenter-counter.js create mode 100644 benchmark/grapheme/bundle-entries/unicode-segmenter-full.js delete mode 100644 src/grapheme-core.js delete mode 100644 src/grapheme-counter.js diff --git a/.changeset/grapheme-counter-entry.md b/.changeset/grapheme-counter-entry.md deleted file mode 100644 index ae081d6..0000000 --- a/.changeset/grapheme-counter-entry.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"unicode-segmenter": minor ---- - -Add `unicode-segmenter/grapheme-counter`, a standalone fast counter for extended grapheme clusters. - -`countGraphemes()` from the new entry returns exactly the same result as the one in `unicode-segmenter/grapheme`, but runs the boundary rules directly without allocating segment objects or slicing strings: - -- 4\~7x faster counting on V8, ~4x on Hermes, with zero allocation -- Standalone entry (4,852 bytes minified, 2,353 bytes gzipped); doesn't carry the segmenter code, and the segmenter entry doesn't carry the counter - even without tree-shaking -- The `countGraphemes()` wrapper in `unicode-segmenter/grapheme` is unchanged and stays as the convenience API diff --git a/benchmark/grapheme/bundle-entries/unicode-segmenter-both.js b/benchmark/grapheme/bundle-entries/unicode-segmenter-both.js deleted file mode 100644 index a96940c..0000000 --- a/benchmark/grapheme/bundle-entries/unicode-segmenter-both.js +++ /dev/null @@ -1,10 +0,0 @@ -// Importing both entries shares `grapheme-core.js`; the counter adds -// only its counting loop on top of the segmenter. -export { - graphemeSegments, - splitGraphemes, - countGraphemes, -} from '../../../src/grapheme.js'; -export { - countGraphemes as countGraphemesFast, -} from '../../../src/grapheme-counter.js'; diff --git a/benchmark/grapheme/bundle-entries/unicode-segmenter-counter.js b/benchmark/grapheme/bundle-entries/unicode-segmenter-counter.js deleted file mode 100644 index 46dc926..0000000 --- a/benchmark/grapheme/bundle-entries/unicode-segmenter-counter.js +++ /dev/null @@ -1,3 +0,0 @@ -export { - countGraphemes, -} from '../../../src/grapheme-counter.js'; diff --git a/benchmark/grapheme/bundle-entries/unicode-segmenter-full.js b/benchmark/grapheme/bundle-entries/unicode-segmenter-full.js new file mode 100644 index 0000000..bdf83dd --- /dev/null +++ b/benchmark/grapheme/bundle-entries/unicode-segmenter-full.js @@ -0,0 +1 @@ +export * from '../../../src/grapheme.js'; diff --git a/benchmark/grapheme/bundle-entries/unicode-segmenter.js b/benchmark/grapheme/bundle-entries/unicode-segmenter.js index ecb7310..2fd1b1c 100644 --- a/benchmark/grapheme/bundle-entries/unicode-segmenter.js +++ b/benchmark/grapheme/bundle-entries/unicode-segmenter.js @@ -1,5 +1,3 @@ export { graphemeSegments, - splitGraphemes, - countGraphemes, } from '../../../src/grapheme.js'; diff --git a/benchmark/grapheme/bundle-stats-hermes.js b/benchmark/grapheme/bundle-stats-hermes.js index 51c5351..521d9fe 100644 --- a/benchmark/grapheme/bundle-stats-hermes.js +++ b/benchmark/grapheme/bundle-stats-hermes.js @@ -18,8 +18,7 @@ let config = await Metro.loadConfig(undefined, { let libs = [ ['unicode-segmenter/grapheme', 'bundle-entries/unicode-segmenter.js'], - ['unicode-segmenter/grapheme-counter', 'bundle-entries/unicode-segmenter-counter.js'], - ['unicode-segmenter/grapheme + grapheme-counter', 'bundle-entries/unicode-segmenter-both.js'], + ['unicode-segmenter/grapheme (full)', 'bundle-entries/unicode-segmenter-full.js'], ['graphemer', 'bundle-entries/graphemer.js'], ['grapheme-splitter', 'bundle-entries/grapheme-splitter.js'], ['@formatjs/intl-segmenter', 'bundle-entries/formatjs-intl-segmenter.js'], diff --git a/benchmark/grapheme/bundle-stats.js b/benchmark/grapheme/bundle-stats.js index 211846e..6a3eba7 100644 --- a/benchmark/grapheme/bundle-stats.js +++ b/benchmark/grapheme/bundle-stats.js @@ -10,8 +10,7 @@ let baseDir = import.meta.dirname; let libs = [ ['unicode-segmenter/grapheme', 'bundle-entries/unicode-segmenter.js'], - ['unicode-segmenter/grapheme-counter', 'bundle-entries/unicode-segmenter-counter.js'], - ['unicode-segmenter/grapheme + grapheme-counter', 'bundle-entries/unicode-segmenter-both.js'], + ['unicode-segmenter/grapheme (full)', 'bundle-entries/unicode-segmenter-full.js'], ['graphemer', 'bundle-entries/graphemer.js'], ['grapheme-splitter', 'bundle-entries/grapheme-splitter.js'], ['@formatjs/intl-segmenter', 'bundle-entries/formatjs-intl-segmenter.js'], diff --git a/benchmark/grapheme/memory-stats.js b/benchmark/grapheme/memory-stats.js index 9fbde6e..9d11481 100644 --- a/benchmark/grapheme/memory-stats.js +++ b/benchmark/grapheme/memory-stats.js @@ -13,7 +13,7 @@ function drain(iterable) { /** @type {Record} */ let libs = { 'unicode-segmenter/grapheme': async () => { - let { graphemeSegments } = await import('../../src/grapheme.js'); + let { graphemeSegments } = await import('unicode-segmenter/grapheme'); return input => drain(graphemeSegments(input)); }, 'graphemer': async () => { diff --git a/package.json b/package.json index 809086f..d1da392 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ "./emoji": "./src/emoji.js", "./general": "./src/general.js", "./grapheme": "./src/grapheme.js", - "./grapheme-counter": "./src/grapheme-counter.js", "./utils": "./src/utils.js", "./intl-adapter": { "types": "./src/intl-adapter.js", @@ -67,10 +66,6 @@ "import": "./grapheme.js", "require": "./grapheme.cjs" }, - "./grapheme-counter": { - "import": "./grapheme-counter.js", - "require": "./grapheme-counter.cjs" - }, "./utils": { "import": "./utils.js", "require": "./utils.cjs" diff --git a/src/grapheme-core.js b/src/grapheme-core.js deleted file mode 100644 index 7cb778b..0000000 --- a/src/grapheme-core.js +++ /dev/null @@ -1,199 +0,0 @@ -// @ts-check - -// Shared machinery for the grapheme segmentation entries -// (`grapheme.js` and `grapheme-counter.js`). -// -// This module is internal; it is not listed in the package exports map -// and its exports are not part of the public API. - -import { decodeUnicodeData, findUnicodeRangeCategory } from './core.js'; -import { grapheme_data, grapheme_cats, grapheme_pairs } from './_grapheme_data.js'; - -/** - * @typedef {import('./_grapheme_data.js').GraphemeCategoryNum} GraphemeCategoryNum - */ - -// Repeated bounds are `let` bindings on purpose; minifiers inline -// `const` numbers into every use site, but keep `let` shared. -export let BMP_MAX = 0xFFFF; -let T1_MIN = 0xA000; -let T2_MIN = 0x1F000; - -// Direct category lookup tables for the hot Unicode regions, and a flat -// binary-search tail for everything rare. -// -// - T0: 0x0000-0x2FFF (ASCII, Latin, Greek, Cyrillic, Semitic, Indic, SEA, ...) -// - 0x3000-0x9FFF (CJK): only 12 non-Any code points, computed inline -// - T1: 0xA000-0xABFF -// - 0xAC00-0xD7A3 (Hangul syllables): LV or LVT computed at runtime -// - 0xD7A4-0xFDFF (Jamo Ext-B, surrogates, private use): computed inline -// - 0xFE00-0xFE0F (variation selectors): computed inline -// - T2: 0x1F000-0x1FAFF (emoji) -// - 0xE0000-0xE0FFF (tags, VS supplement): computed inline -// - everything else: binary search over the range tail (TAIL_S/TAIL_E) -// -// The `Indic_Conjunct_Break=Consonant` property is folded into the category -// space as the 16th category (15) since it never overlaps other categories. -// It shares break semantics with `Any` and never escapes to the public API. -// -// Total index size: ~21KiB of flat typed arrays, no retained JS objects. -const T0 = new Uint8Array(0x3000); -const T1 = new Uint8Array(0xC00); -const T2 = new Uint8Array(0xB00); -/** @type {Uint32Array} Range starts of the binary-search tail */ -let TAIL_S; -/** @type {Uint32Array} Packed `end << 5 | category`, parallel to {@link TAIL_S} */ -let TAIL_E; -{ - let fill = ( - /** @type {Uint8Array} */ t, - /** @type {number} */ lo, - /** @type {number} */ from, - /** @type {number} */ to, - /** @type {number} */ cat, - ) => { - let hi = lo + t.length - 1; - for (let cp = from < lo ? lo : from, top = to > hi ? hi : to; cp <= top; cp++) { - t[cp - lo] = cat; - } - }; - let starts = new Uint32Array(320), ends = new Uint32Array(320), n = 0; - for (let [from, to, cat] of decodeUnicodeData(grapheme_data, grapheme_cats)) { - fill(T0, 0, from, to, cat); - fill(T1, T1_MIN, from, to, cat); - fill(T2, T2_MIN, from, to, cat); - if (to >= 0xFE10 && !(from >= T2_MIN && to <= 0x1FAFF)) { - starts[n] = from; - ends[n++] = to << 5 | cat; - } - } - TAIL_S = starts.slice(0, n); - TAIL_E = ends.slice(0, n); -} - -/** - * `Grapheme_Cluster_Break` property value of a given codepoint - * - * @see https://www.unicode.org/reports/tr29/tr29-45.html#Default_Grapheme_Cluster_Table - * - * @param {number} cp - * @return {number} category number, {@link GraphemeCategoryNum} or 15 (`InCB=Consonant`) - */ -export function cat(cp) { - if (cp < 0x3000) return T0[cp]; - // CJK: 0x3000-0x9FFF - if (cp < T1_MIN) { - if (cp < 0x3030) return cp >= 0x302A ? 3 : 0; - if (cp < 0x309B) { - if (cp === 0x3030 || cp === 0x303D) return 4; - return cp >= 0x3099 ? 3 : 0; - } - return (cp === 0x3297 || cp === 0x3299) ? 4 : 0; - } - if (cp < 0xAC00) return T1[cp - T1_MIN]; - // Hangul syllables: 0xAC00-0xD7A3, LV at every 28th - if (cp < 0xD7A4) return (cp - 0xAC00) % 28 ? 8 : 7; - // Hangul Jamo Extended-B, unassigned and surrogates as Any - if (cp < 0xE000) { - if (cp <= 0xD7C6) return cp >= 0xD7B0 ? 13 : 0; - return (cp >= 0xD7CB && cp <= 0xD7FB) ? 12 : 0; - } - // Private use - if (cp < 0xFE00) return cp === 0xFB1E ? 3 : 0; - // Variation selectors - if (cp < 0xFE10) return 3; - // Emoji: 0x1F000-0x1FAFF - if (cp >= T2_MIN && cp < 0x1FB00) return T2[cp - T2_MIN]; - // Tags and variation selectors supplement: 0xE0000-0xE0FFF - if (cp >= 0xE0000) { - if (cp > 0xE0FFF) return 0; - return ((cp >= 0xE0020 && cp < 0xE0080) || (cp >= 0xE0100 && cp < 0xE01F0)) ? 3 : 2; - } - // The rare tail: 0xFE10-0xFFFF, 0x10000-0x1EFFF, 0x1FB00-0xDFFFF - return findUnicodeRangeCategory(cp, TAIL_S, TAIL_E); -} - -// Boundary decision table for category pairs, `PAIR[catBefore << 4 | catAfter]` -// -// - 0: boundary (GB999 and friends) -// - 1: no boundary -// - 2: GB12/GB13, no boundary iff odd run of RI precedes -// - 3: GB11, no boundary iff the ZWJ was preceded by ExtPic Extend* -// - 4: GB9c, no boundary iff InCB Consonant [Extend Linker]* Linker [Extend Linker]* precedes -export const PAIR = Uint8Array.from(grapheme_pairs, Number); - -/** - * The Unicode `Indic_Conjunct_Break=Linker` set - * - * @param {number} cp - * @return {boolean} - */ -function isLinker(cp) { - return cp === 0x094D // Devanagari Sign Virama - || cp === 0x09CD // Bengali Sign Virama - || cp === 0x0A4D // Gurmukhi Sign Virama - || cp === 0x0ACD // Gujarati Sign Virama - || cp === 0x0B4D // Oriya Sign Virama - || cp === 0x0C4D // Telugu Sign Virama - || cp === 0x0D4D // Malayalam Sign Virama - || cp === 0x1039 // Myanmar Sign Virama - || cp === 0x17D2 // Khmer Sign Coeng - || cp === 0x1A60 // Tai Tham Sign Sakot - || cp === 0x1B44 // Balinese Adeg Adeg - || cp === 0x1BAB // Sundanese Sign Virama - || cp === 0xA9C0 // Javanese Pangkon - || cp === 0xAAF6 // Meetei Mayek Virama - || cp === 0x10A3F // Kharoshthi Virama - || cp === 0x11133 // Chakma Virama - || cp === 0x113D0 // Tulu-Tigalari Conjoiner - || cp === 0x1193E // Dives Akuru Virama - || cp === 0x11A47 // Zanabazar Square Subjoiner - || cp === 0x11A99 // Soyombo Subjoiner - || cp === 0x11F42; // Kawi Conjoiner -} - -// Sequence state, packed in a small int: -// -// bit 0 : odd run of Regional_Indicator immediately precedes (GB12, GB13) -// bit 1 : ExtPic Extend* immediately precedes (GB11) -// bit 2 : the last consumed ZWJ was preceded by ExtPic Extend* (GB11) -// bit 3-4 : InCB state; 01 = Consonant [Extend Linker]* precedes, -// 10 = it also contains a Linker (GB9c) -// -// It is a pure function of the consumed code point sequence, so it carries -// across segment boundaries without any reset. Callers skip the transition -// (resetting state to 0) unless the category can arm or keep state; -// see the `0xC418` masks (categories 3, 4, 10, 14, and 15) on call sites. - -/** - * State transition on consuming a code point. - * - * @param {number} st packed state - * @param {number} c category of the consumed code point - * @param {number} cp the consumed code point - * @return {number} next packed state - */ -export function nextState(st, c, cp) { - switch (c) { - // Extend; keeps picto bits, advances InCB run - case 3: - if (st & 24) { - if (cp === 0x200C) return st & 6; // ZWNJ has InCB=None - if ((st & 24) === 16 || isLinker(cp)) return (st & 6) | 16; - return (st & 6) | 8; - } - return st & 6; - // Extended_Pictographic - case 4: - return 2; - // Regional_Indicator; toggles parity - case 10: - return (st & 1) ^ 1; - // ZWJ; captures the picto bit, advances InCB run - case 14: - return (st & 2) << 1 | (st & 24); - // InCB=Consonant (15); callers never pass other categories - default: - return 8; - } -} diff --git a/src/grapheme-counter.js b/src/grapheme-counter.js deleted file mode 100644 index d0f111c..0000000 --- a/src/grapheme-counter.js +++ /dev/null @@ -1,56 +0,0 @@ -// @ts-check - -import { PAIR, cat, nextState } from './grapheme-core.js'; - -/** - * Count number of extended grapheme clusters in given text. - * - * The result is identical to `countGraphemes` of the - * `unicode-segmenter/grapheme` entry, but this one runs the boundary - * rules directly without allocating segment objects or slicing strings, - * so counting is significantly faster. - * - * It is a standalone entry so that counting doesn't have to carry the - * segmenter code, and vice versa - even without tree-shaking. - * - * @param {string} text - * @return {number} count of grapheme clusters - */ -export function countGraphemes(text) { - let len = text.length; - if (len === 0) return 0; - - let cp = /** @type {number} */ (text.codePointAt(0)); - let cursor = cp > 0xFFFF ? 2 : 1; - - /** Category of the last consumed code point */ - let catBefore = cat(cp); - - /** Packed sequence state; see `grapheme-core.js` */ - let st = (0xC418 >> catBefore) & 1 ? nextState(0, catBefore, cp) : 0; - - /** The segment being scanned counts, whether or not a boundary follows */ - let count = 1; - - while (cursor < len) { - cp = /** @type {number} */ (text.codePointAt(cursor)); - let catAfter = cat(cp); - let d = PAIR[catBefore << 4 | catAfter]; - let boundary; - if (d === 0) boundary = true; - else if (d === 1) boundary = false; - else if (d === 2) boundary = !(st & 1); - else if (d === 3) boundary = !(st & 4); - else boundary = (st & 24) !== 16; - - st = (0xC418 >> catAfter) & 1 && (st !== 0 || catAfter !== 3) - ? nextState(st, catAfter, cp) - : 0; - - if (boundary) count += 1; - cursor += cp > 0xFFFF ? 2 : 1; - catBefore = catAfter; - } - - return count; -} diff --git a/src/grapheme.js b/src/grapheme.js index 7f3afde..9b34838 100644 --- a/src/grapheme.js +++ b/src/grapheme.js @@ -1,7 +1,12 @@ // @ts-check -import { PAIR, cat, nextState } from './grapheme-core.js'; -import { GraphemeCategory } from './_grapheme_data.js'; +import { decodeUnicodeData, findUnicodeRangeCategory } from './core.js'; +import { + GraphemeCategory, + grapheme_cats, + grapheme_data, + grapheme_pairs, +} from './_grapheme_data.js'; /** * @typedef {import('./_grapheme_data.js').GC_Any} GC_Any @@ -19,6 +24,191 @@ import { GraphemeCategory } from './_grapheme_data.js'; export { GraphemeCategory }; +// Repeated bounds are `let` bindings on purpose; minifiers inline +// `const` numbers into every use site, but keep `let` shared. +let BMP_MAX = 0xFFFF; +let T1_MIN = 0xA000; +let T2_MIN = 0x1F000; + +// Direct category lookup tables for the hot Unicode regions, and a flat +// binary-search tail for everything rare. +// +// - T0: 0x0000-0x2FFF (ASCII, Latin, Greek, Cyrillic, Semitic, Indic, SEA, ...) +// - 0x3000-0x9FFF (CJK): only 12 non-Any code points, computed inline +// - T1: 0xA000-0xABFF +// - 0xAC00-0xD7A3 (Hangul syllables): LV or LVT computed at runtime +// - 0xD7A4-0xFDFF (Jamo Ext-B, surrogates, private use): computed inline +// - 0xFE00-0xFE0F (variation selectors): computed inline +// - T2: 0x1F000-0x1FAFF (emoji) +// - 0xE0000-0xE0FFF (tags, VS supplement): computed inline +// - everything else: binary search over the range tail (TAIL_S/TAIL_E) +// +// The `Indic_Conjunct_Break=Consonant` property is folded into the category +// space as the 16th category (15) since it never overlaps other categories. +// It shares break semantics with `Any` and never escapes to the public API. +// +// Total index size: ~21KiB of flat typed arrays, no retained JS objects. +const T0 = new Uint8Array(0x3000); +const T1 = new Uint8Array(0xC00); +const T2 = new Uint8Array(0xB00); +/** @type {Uint32Array} Range starts of the binary-search tail */ +let TAIL_S; +/** @type {Uint32Array} Packed `end << 5 | category`, parallel to {@link TAIL_S} */ +let TAIL_E; +{ + let fill = ( + /** @type {Uint8Array} */ t, + /** @type {number} */ lo, + /** @type {number} */ from, + /** @type {number} */ to, + /** @type {number} */ cat, + ) => { + let hi = lo + t.length - 1; + for (let cp = from < lo ? lo : from, top = to > hi ? hi : to; cp <= top; cp++) { + t[cp - lo] = cat; + } + }; + let starts = new Uint32Array(320), ends = new Uint32Array(320), n = 0; + for (let [from, to, cat] of decodeUnicodeData(grapheme_data, grapheme_cats)) { + fill(T0, 0, from, to, cat); + fill(T1, T1_MIN, from, to, cat); + fill(T2, T2_MIN, from, to, cat); + if (to >= 0xFE10 && !(from >= T2_MIN && to <= 0x1FAFF)) { + starts[n] = from; + ends[n++] = to << 5 | cat; + } + } + TAIL_S = starts.slice(0, n); + TAIL_E = ends.slice(0, n); +} + +/** + * `Grapheme_Cluster_Break` property value of a given codepoint + * + * @see https://www.unicode.org/reports/tr29/tr29-45.html#Default_Grapheme_Cluster_Table + * + * @param {number} cp + * @return {number} category number, {@link GraphemeCategoryNum} or 15 (`InCB=Consonant`) + */ +export function cat(cp) { + if (cp < 0x3000) return T0[cp]; + // CJK: 0x3000-0x9FFF + if (cp < T1_MIN) { + if (cp < 0x3030) return cp >= 0x302A ? 3 : 0; + if (cp < 0x309B) { + if (cp === 0x3030 || cp === 0x303D) return 4; + return cp >= 0x3099 ? 3 : 0; + } + return (cp === 0x3297 || cp === 0x3299) ? 4 : 0; + } + if (cp < 0xAC00) return T1[cp - T1_MIN]; + // Hangul syllables: 0xAC00-0xD7A3, LV at every 28th + if (cp < 0xD7A4) return (cp - 0xAC00) % 28 ? 8 : 7; + // Hangul Jamo Extended-B, unassigned and surrogates as Any + if (cp < 0xE000) { + if (cp <= 0xD7C6) return cp >= 0xD7B0 ? 13 : 0; + return (cp >= 0xD7CB && cp <= 0xD7FB) ? 12 : 0; + } + // Private use + if (cp < 0xFE00) return cp === 0xFB1E ? 3 : 0; + // Variation selectors + if (cp < 0xFE10) return 3; + // Emoji: 0x1F000-0x1FAFF + if (cp >= T2_MIN && cp < 0x1FB00) return T2[cp - T2_MIN]; + // Tags and variation selectors supplement: 0xE0000-0xE0FFF + if (cp >= 0xE0000) { + if (cp > 0xE0FFF) return 0; + return ((cp >= 0xE0020 && cp < 0xE0080) || (cp >= 0xE0100 && cp < 0xE01F0)) ? 3 : 2; + } + // The rare tail: 0xFE10-0xFFFF, 0x10000-0x1EFFF, 0x1FB00-0xDFFFF + return findUnicodeRangeCategory(cp, TAIL_S, TAIL_E); +} + +// Boundary decision table for category pairs, `PAIR[catBefore << 4 | catAfter]` +// +// - 0: boundary (GB999 and friends) +// - 1: no boundary +// - 2: GB12/GB13, no boundary iff odd run of RI precedes +// - 3: GB11, no boundary iff the ZWJ was preceded by ExtPic Extend* +// - 4: GB9c, no boundary iff InCB Consonant [Extend Linker]* Linker [Extend Linker]* precedes +export const PAIR = Uint8Array.from(grapheme_pairs, Number); + +/** + * The Unicode `Indic_Conjunct_Break=Linker` set + * + * @param {number} cp + * @return {boolean} + */ +function isLinker(cp) { + return cp === 0x094D // Devanagari Sign Virama + || cp === 0x09CD // Bengali Sign Virama + || cp === 0x0A4D // Gurmukhi Sign Virama + || cp === 0x0ACD // Gujarati Sign Virama + || cp === 0x0B4D // Oriya Sign Virama + || cp === 0x0C4D // Telugu Sign Virama + || cp === 0x0D4D // Malayalam Sign Virama + || cp === 0x1039 // Myanmar Sign Virama + || cp === 0x17D2 // Khmer Sign Coeng + || cp === 0x1A60 // Tai Tham Sign Sakot + || cp === 0x1B44 // Balinese Adeg Adeg + || cp === 0x1BAB // Sundanese Sign Virama + || cp === 0xA9C0 // Javanese Pangkon + || cp === 0xAAF6 // Meetei Mayek Virama + || cp === 0x10A3F // Kharoshthi Virama + || cp === 0x11133 // Chakma Virama + || cp === 0x113D0 // Tulu-Tigalari Conjoiner + || cp === 0x1193E // Dives Akuru Virama + || cp === 0x11A47 // Zanabazar Square Subjoiner + || cp === 0x11A99 // Soyombo Subjoiner + || cp === 0x11F42; // Kawi Conjoiner +} + +// Sequence state, packed in a small int: +// +// bit 0 : odd run of Regional_Indicator immediately precedes (GB12, GB13) +// bit 1 : ExtPic Extend* immediately precedes (GB11) +// bit 2 : the last consumed ZWJ was preceded by ExtPic Extend* (GB11) +// bit 3-4 : InCB state; 01 = Consonant [Extend Linker]* precedes, +// 10 = it also contains a Linker (GB9c) +// +// It is a pure function of the consumed code point sequence, so it carries +// across segment boundaries without any reset. Callers skip the transition +// (resetting state to 0) unless the category can arm or keep state; +// see the `0xC418` masks (categories 3, 4, 10, 14, and 15) on call sites. + +/** + * State transition on consuming a code point. + * + * @param {number} st packed state + * @param {number} c category of the consumed code point + * @param {number} cp the consumed code point + * @return {number} next packed state + */ +export function nextState(st, c, cp) { + switch (c) { + // Extend; keeps picto bits, advances InCB run + case 3: + if (st & 24) { + if (cp === 0x200C) return st & 6; // ZWNJ has InCB=None + if ((st & 24) === 16 || isLinker(cp)) return (st & 6) | 16; + return (st & 6) | 8; + } + return st & 6; + // Extended_Pictographic + case 4: + return 2; + // Regional_Indicator; toggles parity + case 10: + return (st & 1) ^ 1; + // ZWJ; captures the picto bit, advances InCB run + case 14: + return (st & 2) << 1 | (st & 24); + // InCB=Consonant (15); callers never pass other categories + default: + return 8; + } +} + /** * Unicode segmentation by extended grapheme rules. * @@ -33,7 +223,7 @@ export function* graphemeSegments(input) { if (len === 0) return; let cp = /** @type {number} */ (input.codePointAt(0)); - let cursor = cp > 0xFFFF ? 2 : 1; + let cursor = cp > BMP_MAX ? 2 : 1; /** Category of the last consumed code point */ let catBefore = cat(cp); @@ -78,7 +268,7 @@ export function* graphemeSegments(input) { hd = cp; catBegin = catAfter; } - cursor += cp > 0xFFFF ? 2 : 1; + cursor += cp > BMP_MAX ? 2 : 1; catBefore = catAfter; } diff --git a/test/corpora.js b/test/corpora.js index 2546a6b..e904b64 100644 --- a/test/corpora.js +++ b/test/corpora.js @@ -9,9 +9,6 @@ import { splitGraphemes, countGraphemes, } from 'unicode-segmenter/grapheme'; -import { - countGraphemes as countGraphemesFast, -} from 'unicode-segmenter/grapheme-counter'; // Real-world text fixtures generated by `scripts/corpora.js`. // @@ -82,7 +79,6 @@ test('corpora', async t => { // Bind the other public APIs to the same corpus. assert.equal(countGraphemes(text), expected.length); - assert.equal(countGraphemesFast(text), expected.length); let split = [...splitGraphemes(text)]; assert.equal(split.length, expected.length); diff --git a/test/grapheme.js b/test/grapheme.js index 61df057..5bac9ec 100644 --- a/test/grapheme.js +++ b/test/grapheme.js @@ -10,9 +10,6 @@ import { splitGraphemes, countGraphemes, } from 'unicode-segmenter/grapheme'; -import { - countGraphemes as countGraphemesFast, -} from 'unicode-segmenter/grapheme-counter'; import { assertObjectContaining } from './_helper.js'; test('graphemeSegments', async t => { @@ -80,39 +77,31 @@ test('graphemeSegments', async t => { test('countGraphemes', async t => { await t.test('latin', () => { assert.equal(countGraphemes('abcd'), 4); - assert.equal(countGraphemesFast('abcd'), 4); }); await t.test('flags', () => { assert.equal(countGraphemes('🇷🇸🇮🇴'), 2); - assert.equal(countGraphemesFast('🇷🇸🇮🇴'), 2); }); await t.test('emoji', () => { assert.equal(countGraphemes('👻👩‍👩‍👦‍👦'), 2); - assert.equal(countGraphemesFast('👻👩‍👩‍👦‍👦'), 2); assert.equal(countGraphemes('🌷🎁💩😜👍🏳️‍🌈'), 6); - assert.equal(countGraphemesFast('🌷🎁💩😜👍🏳️‍🌈'), 6); }); await t.test('diacritics as combining marks', () => { assert.equal(countGraphemes('Ĺo͂řȩm̅'), 5); - assert.equal(countGraphemesFast('Ĺo͂řȩm̅'), 5); }); await t.test('Jamo', () => { assert.equal(countGraphemes('뎌쉐'), 2); - assert.equal(countGraphemesFast('뎌쉐'), 2); }); await t.test('Hindi', () => { assert.equal(countGraphemes('अनुच्छेद'), 4); - assert.equal(countGraphemesFast('अनुच्छेद'), 4); }); await t.test('demonic', () => { assert.equal(countGraphemes('Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'), 6); - assert.equal(countGraphemesFast('Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'), 6); }); }); @@ -184,7 +173,8 @@ test('conformance', async t => { let assertProperty = input => { let expected = [...intlSegmenter.segment(input)]; assertObjectContaining([...graphemeSegments(input)], expected); - assert.equal(countGraphemesFast(input), expected.length); + assert.equal(countGraphemes(input), expected.length); + assert.deepEqual([...splitGraphemes(input)], expected.map(s => s.segment)); } await t.test('any string', () => { From 291e20f03b2174a4df1bdca9676294bd7daaeca0 Mon Sep 17 00:00:00 2001 From: Hyeseong Kim Date: Wed, 22 Jul 2026 16:21:32 +0900 Subject: [PATCH 2/4] update README stats section --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 377c6f4..d97184b 100644 --- a/README.md +++ b/README.md @@ -244,15 +244,15 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb | Name | Unicode® | ESM? | Size | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) | |-------------------------------------------------|----------|------|--------:|-----------:|----------------:|--------------:|----------------:| -| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 9,009 | 5,171 | 2,497 | 2,249 | 2,544 | -| `unicode-segmenter/grapheme-counter` | 17.0.0 | ✔️ | 7,912 | 4,852 | 2,353 | 2,120 | 2,396 | -| `unicode-segmenter/grapheme + grapheme-counter` | 17.0.0 | ✔️ | 9,985 | 5,489 | 2,571 | 2,310 | 2,612 | +| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,203 | 2,507 | +| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 9,268 | 5,313 | 2,624 | 2,350 | 2,672 | | `graphemer` | 15.0.0 | ✖️ ️ | 410,435 | 95,104 | 15,752 | 10,660 | 15,911 | | `grapheme-splitter` | 10.0.0 | ✖️ | 122,254 | 23,682 | 7,852 | 4,802 | 6,753 | | `@formatjs/intl-segmenter`* | 17.0.0 | ✖️ | 268,301 | 176,759 | 45,988 | 31,701 | 45,370 | | `unicode-segmentation`* | 15.1.0 | - | 56,529 | 52,439 | 24,108 | 17,343 | 24,375 | | `Intl.Segmenter`* | - | - | 0 | 0 | 0 | 0 | 0 | +* `unicode-segmenter/grapheme` provides count/split-only API which has much less overhead than full `Segmenter` implementation, which includes some redundant codes but tree-shakable. * `@formatjs/intl-segmenter` handles grapheme, word, and sentence, but it's not tree-shakable. * `unicode-segmentation` size contains only minimum WASM binary and its bindings to execute benchmarking. It will increases to expose more features. * `Intl.Segmenter`'s Unicode data depends on the host, and may not be up-to-date. @@ -262,9 +262,8 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb | Name | Bytecode size | Bytecode size (gzip)* | |-------------------------------------------------|--------------:|----------------------:| -| `unicode-segmenter/grapheme` | 19,721 | 10,874 | -| `unicode-segmenter/grapheme-counter` | 18,210 | 10,069 | -| `unicode-segmenter/grapheme + grapheme-counter` | 20,667 | 11,371 | +| `unicode-segmenter/grapheme` | 19,196 | 10,715 | +| `unicode-segmenter/grapheme` (full) | 19,401 | 10,848 | | `graphemer` | 134,085 | 31,770 | | `grapheme-splitter` | 63,942 | 19,165 | | `@formatjs/intl-segmenter` | 329,547 | 136,751 | From d11176cdc875ad7e20897b066f18259959c271e7 Mon Sep 17 00:00:00 2001 From: Hyeseong Kim Date: Wed, 22 Jul 2026 16:22:44 +0900 Subject: [PATCH 3/4] remove old section --- README.md | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/README.md b/README.md index d97184b..8e62d35 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,6 @@ countGraphemes('a̐éö̲'); > `countGraphemes()` is a small wrapper around `graphemeSegments()`. > > If you need it more than once at a time, consider memoization or use `graphemeSegments()` or `splitGraphemes()` once instead. -> -> If counting is on your hot path, use the standalone [`unicode-segmenter/grapheme-counter`](#export-unicode-segmentergrapheme-counter) entry instead. #### Example: Build an advanced grapheme matcher @@ -125,31 +123,6 @@ function* matchEmoji(str) { Or build even more advanced one like an Unicode-aware [TTY string width](https://github.com/cometkim/unicode-string-width) utility. -### Export `unicode-segmenter/grapheme-counter` - -A standalone counter for extended grapheme clusters. - -The result is identical to `countGraphemes()` of `unicode-segmenter/grapheme`, but it runs the boundary rules directly without allocating segment objects or slicing strings, so counting is 4+ times faster and zero heap usage. - -It is a separate entry so that counting doesn't have to carry the segmenter code, and vice versa. - -#### Example: Count graphemes fast - -```js -import { countGraphemes } from 'unicode-segmenter/grapheme-counter'; - -countGraphemes('👋 안녕!'); -// => 5 -``` - -> [!NOTE] -> The counter module contains duplicate code from `graphemeSegments()`. -> -> If you want a smaller bundle size and performance multipliers are not very important, use the `unicode-segmenter/grapheme` module instead. -> -> Full Segmenter is already optimized enough. - - ### Export `unicode-segmenter/intl-adapter` [`Intl.Segmenter`] API adapter (only `granularity: "grapheme"` available yet) From a43e3e73119b59e332c2af494ca6a8c335e6be8e Mon Sep 17 00:00:00 2001 From: Hyeseong Kim Date: Wed, 22 Jul 2026 16:58:49 +0900 Subject: [PATCH 4/4] make bundle-stats printer result copyable --- README.md | 32 +++++++++++------------ benchmark/_helper.js | 26 +++++++++--------- benchmark/grapheme/bundle-stats-hermes.js | 10 +++---- benchmark/grapheme/bundle-stats.js | 12 ++++----- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 8e62d35..06ab3de 100644 --- a/README.md +++ b/README.md @@ -215,15 +215,15 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb #### JS Bundle Stats -| Name | Unicode® | ESM? | Size | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) | -|-------------------------------------------------|----------|------|--------:|-----------:|----------------:|--------------:|----------------:| -| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,203 | 2,507 | -| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 9,268 | 5,313 | 2,624 | 2,350 | 2,672 | -| `graphemer` | 15.0.0 | ✖️ ️ | 410,435 | 95,104 | 15,752 | 10,660 | 15,911 | -| `grapheme-splitter` | 10.0.0 | ✖️ | 122,254 | 23,682 | 7,852 | 4,802 | 6,753 | -| `@formatjs/intl-segmenter`* | 17.0.0 | ✖️ | 268,301 | 176,759 | 45,988 | 31,701 | 45,370 | -| `unicode-segmentation`* | 15.1.0 | - | 56,529 | 52,439 | 24,108 | 17,343 | 24,375 | -| `Intl.Segmenter`* | - | - | 0 | 0 | 0 | 0 | 0 | +| Name | Unicode® | ESM? | Size | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) | +|--------------------------------------|----------|------|--------:|-----------:|----------------:|--------------:|----------------:| +| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,203 | 2,507 | +| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 9,268 | 5,313 | 2,624 | 2,350 | 2,672 | +| `graphemer` | 15.0.0 | ✖️ | 410,435 | 95,104 | 15,752 | 10,660 | 15,911 | +| `grapheme-splitter` | 10.0.0 | ✖️ | 122,254 | 23,682 | 7,852 | 4,802 | 6,753 | +| `@formatjs/intl-segmenter`* | 17.0.0 | ✖️ | 268,301 | 176,759 | 45,988 | 31,701 | 45,370 | +| `unicode-segmentation`* | 15.1.0 | - | 56,529 | 52,439 | 24,108 | 17,343 | 24,375 | +| `Intl.Segmenter`* | - | - | 0 | 0 | 0 | 0 | 0 | * `unicode-segmenter/grapheme` provides count/split-only API which has much less overhead than full `Segmenter` implementation, which includes some redundant codes but tree-shakable. * `@formatjs/intl-segmenter` handles grapheme, word, and sentence, but it's not tree-shakable. @@ -233,13 +233,13 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb #### Hermes Bytecode Stats -| Name | Bytecode size | Bytecode size (gzip)* | -|-------------------------------------------------|--------------:|----------------------:| -| `unicode-segmenter/grapheme` | 19,196 | 10,715 | -| `unicode-segmenter/grapheme` (full) | 19,401 | 10,848 | -| `graphemer` | 134,085 | 31,770 | -| `grapheme-splitter` | 63,942 | 19,165 | -| `@formatjs/intl-segmenter` | 329,547 | 136,751 | +| Name | Bytecode size | Bytecode size (gzip)* | +|-------------------------------------|--------------:|----------------------:| +| `unicode-segmenter/grapheme` | 19,196 | 10,715 | +| `unicode-segmenter/grapheme` (full) | 19,401 | 10,848 | +| `graphemer` | 134,085 | 31,770 | +| `grapheme-splitter` | 63,942 | 19,165 | +| `@formatjs/intl-segmenter` | 329,547 | 136,751 | * The installation size contains _compressed_ assets. diff --git a/benchmark/_helper.js b/benchmark/_helper.js index 9cca8b6..33b2f3a 100644 --- a/benchmark/_helper.js +++ b/benchmark/_helper.js @@ -70,7 +70,7 @@ export function printStats(reports) { let value = report[key]; if (value == null) return ''; if (typeof value === 'number') return value.toLocaleString('en-US'); - return i === 0 ? `\`${value}\`` : String(value); + return i === 0 ? `${value}` : String(value); })); let widths = columns.map((key, i) => { @@ -92,10 +92,13 @@ export function printStats(reports) { * `UPDATE_README=true`. * * The table is located by its `####
` heading. Rows are - * matched by library name and columns by header text, both ignoring - * the `*` footnote marks, which are preserved as-is. Cells with no - * matching report column (e.g. Unicode®, ESM?) and rows with no - * matching report (e.g. `Intl.Segmenter`) are left untouched. + * matched by their first cell against `report.name` verbatim — names + * are expected to already contain any backticks/footnote marks (e.g. + * `` `unicode-segmenter/grapheme` (full*) ``), so the README row's + * name cell is preserved as-is. Columns are matched by header text + * with `*` footnote marks stripped. Cells with no matching report + * column (e.g. Unicode®, ESM?) and rows with no matching report + * (e.g. `Intl.Segmenter`) are left untouched. * * @param {Array>} reports * @param {string} section heading text, e.g. 'JS Bundle Stats' @@ -121,7 +124,10 @@ export async function updateReadmeStats(reports, section) { let end = head + 1; while (end < lines.length && lines[end].trimStart().startsWith('|')) end += 1; - let split = line => line.trim().replace(/^\||\|$/g, '').split('|').map(cell => cell.trim()); + let split = line => line.trim().replace(/^\||\|$/g, '').split('|').map(cell => cell + .trim() + .replace(/\s+[\uFE0F\uFE0E]+$/, '') + .trim()); let stripMark = text => text.replace(/\*+$/, '').trim(); let header = split(lines[head]); @@ -132,16 +138,10 @@ export async function updateReadmeStats(reports, section) { let colKeys = header.map(stripMark); let byName = new Map(reports.map(report => [String(report.name), report])); - for (let report of reports) { - // e.g. the 'unicode-segmentation (wasm-bindgen)' report matches - // the `unicode-segmentation` row - let alias = String(report.name).replace(/\s*\([^)]*\)$/, ''); - if (!byName.has(alias)) byName.set(alias, report); - } let matched = new Set(); for (let row of body) { - let report = byName.get(stripMark(row[0]).replace(/`/g, '').trim()); + let report = byName.get(row[0]); if (!report) continue; matched.add(report); for (let i = 1; i < header.length; i++) { diff --git a/benchmark/grapheme/bundle-stats-hermes.js b/benchmark/grapheme/bundle-stats-hermes.js index 521d9fe..7b2c36c 100644 --- a/benchmark/grapheme/bundle-stats-hermes.js +++ b/benchmark/grapheme/bundle-stats-hermes.js @@ -17,11 +17,11 @@ let config = await Metro.loadConfig(undefined, { }); let libs = [ - ['unicode-segmenter/grapheme', 'bundle-entries/unicode-segmenter.js'], - ['unicode-segmenter/grapheme (full)', 'bundle-entries/unicode-segmenter-full.js'], - ['graphemer', 'bundle-entries/graphemer.js'], - ['grapheme-splitter', 'bundle-entries/grapheme-splitter.js'], - ['@formatjs/intl-segmenter', 'bundle-entries/formatjs-intl-segmenter.js'], + ['`unicode-segmenter/grapheme`', 'bundle-entries/unicode-segmenter.js'], + ['`unicode-segmenter/grapheme` (full)', 'bundle-entries/unicode-segmenter-full.js'], + ['`graphemer`', 'bundle-entries/graphemer.js'], + ['`grapheme-splitter`', 'bundle-entries/grapheme-splitter.js'], + ['`@formatjs/intl-segmenter`', 'bundle-entries/formatjs-intl-segmenter.js'], ]; let reports = []; diff --git a/benchmark/grapheme/bundle-stats.js b/benchmark/grapheme/bundle-stats.js index 6a3eba7..2d659ad 100644 --- a/benchmark/grapheme/bundle-stats.js +++ b/benchmark/grapheme/bundle-stats.js @@ -9,11 +9,11 @@ import { printStats, reportBundleStats, reportWasmBindingStats, updateReadmeStat let baseDir = import.meta.dirname; let libs = [ - ['unicode-segmenter/grapheme', 'bundle-entries/unicode-segmenter.js'], - ['unicode-segmenter/grapheme (full)', 'bundle-entries/unicode-segmenter-full.js'], - ['graphemer', 'bundle-entries/graphemer.js'], - ['grapheme-splitter', 'bundle-entries/grapheme-splitter.js'], - ['@formatjs/intl-segmenter', 'bundle-entries/formatjs-intl-segmenter.js'], + ['`unicode-segmenter/grapheme`', 'bundle-entries/unicode-segmenter.js'], + ['`unicode-segmenter/grapheme` (full*)', 'bundle-entries/unicode-segmenter-full.js'], + ['`graphemer`', 'bundle-entries/graphemer.js'], + ['`grapheme-splitter`', 'bundle-entries/grapheme-splitter.js'], + ['`@formatjs/intl-segmenter`*', 'bundle-entries/formatjs-intl-segmenter.js'], ]; let reports = await Promise.all( @@ -40,7 +40,7 @@ let reports = await Promise.all( ); { - let libName = 'unicode-segmentation (wasm-bindgen)'; + let libName = '`unicode-segmentation`*'; let bindingEntry = path.join(baseDir, 'bundle-entries/unicode-segmentation.js'); let bindingResult = await build({ write: false,