diff --git a/.changeset/heavy-grapes-accept.md b/.changeset/heavy-grapes-accept.md new file mode 100644 index 0000000..59d90f9 --- /dev/null +++ b/.changeset/heavy-grapes-accept.md @@ -0,0 +1,9 @@ +--- +"unicode-segmenter": patch +--- + +Add `collectGraphemes(input)` API that collects grapheme clusters into an array directly. + +This is a fast version of `[...splitGraphemes(input)]`, which is acually the most used pattern in practice. + +It's 2-4x faster than the iterator-based approach. However, it collects all grapheme clusters at once, so it's not good for large text or streamed input. diff --git a/README.md b/README.md index c5fb275..426713b 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Utilities for text segmentation by extended grapheme cluster rules. You can retrieve all grapheme segments using the `graphemeSegments()` generator. -Each segment is returned as a `GraphemeSegment` object containing the `segment` (the substring), the starting `index`, and the full `input` string. This pattern is similar to the [`Intl.Segmenter`] API, so it can be a drop-in replacement. +Each segment is returned as a `GraphemeSegmentOutput` object containing the `segment` (the substring), the starting `index`, and the full `input` string. This pattern is similar to the [`Intl.Segmenter`] API, so it can be a drop-in replacement. ```js import { graphemeSegments } from 'unicode-segmenter/grapheme'; @@ -81,6 +81,20 @@ import { splitGraphemes } from 'unicode-segmenter/grapheme'; // 4: 2️⃣ ``` +#### Example: Collect graphemes into an array + +If you need the result as an array, use `collectGraphemes()` for convenience. + +This is a fast version of `[...splitGraphemes(str)]`, 2-4x faster. + +```js +import { collectGraphemes } from 'unicode-segmenter/grapheme'; + +collectGraphemes('#️⃣*️⃣0️⃣1️⃣2️⃣'); // => ["#️⃣", "*️⃣", "0️⃣", "1️⃣", "2️⃣"] +``` + +However, it collects all grapheme clusters at once, so it's not good for large text or streamed input. + #### Example: Count graphemes If you need only the count, use `countGraphemes()`, which is optimized to avoid allocations for segments. @@ -223,8 +237,8 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb | Name | Unicode® | ESM? | Size | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) | |--------------------------------------|----------|------|--------:|-----------:|----------------:|--------------:|----------------:| -| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,202 | 2,507 | -| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 10,092 | 5,573 | 2,681 | 2,402 | 2,722 | +| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,218 | 2,508 | +| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 11,182 | 5,927 | 2,820 | 2,492 | 2,865 | | `graphemer` | 15.0.0 | ✖️ | 410,435 | 95,104 | 15,752 | 10,660 | 15,911 | | `grapheme-splitter` | 10.0.0 | ✖️ | 122,254 | 23,682 | 7,852 | 4,802 | 6,753 | | `@formatjs/intl-segmenter`* | 17.0.0 | ✖️ | 268,301 | 176,759 | 45,988 | 31,701 | 45,370 | @@ -241,8 +255,8 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb | Name | Bytecode size | Bytecode size (gzip)* | |-------------------------------------|--------------:|----------------------:| -| `unicode-segmenter/grapheme` | 19,538 | 10,957 | -| `unicode-segmenter/grapheme` (full) | 19,743 | 11,080 | +| `unicode-segmenter/grapheme` | 20,203 | 11,224 | +| `unicode-segmenter/grapheme` (full) | 20,408 | 11,356 | | `graphemer` | 134,085 | 31,770 | | `grapheme-splitter` | 63,942 | 19,165 | | `@formatjs/intl-segmenter` | 329,547 | 136,751 | diff --git a/benchmark/grapheme/codspeed.js b/benchmark/grapheme/codspeed.js index 9bf18a8..925426e 100644 --- a/benchmark/grapheme/codspeed.js +++ b/benchmark/grapheme/codspeed.js @@ -5,6 +5,7 @@ import { graphemeSegments, countGraphemes, splitGraphemes, + collectGraphemes, } from 'unicode-segmenter/grapheme'; import { testcases } from './_testcases.js'; @@ -50,5 +51,17 @@ for (const [name, input] of testcases) { }); } +for (const [name, input] of testcases) { + bench.add(`collectGraphemes - ${name}`, () => { + void collectGraphemes(input); + }, { + beforeAll() { + for (let i = 0; i < 2000; i++) { + void collectGraphemes(input); + } + }, + }); +} + await bench.run(); console.table(bench.table()); diff --git a/benchmark/grapheme/perf-collect.js b/benchmark/grapheme/perf-collect.js new file mode 100644 index 0000000..14c46a9 --- /dev/null +++ b/benchmark/grapheme/perf-collect.js @@ -0,0 +1,57 @@ +import { + group, + summary, + barplot, + bench, + run, + do_not_optimize, +} from 'mitata'; + +import { + splitGraphemes, + collectGraphemes, +} from '../../src/grapheme.js'; +import { testcases } from './_testcases.js'; + +// Node.js, Deno, Bun +const isSystemRuntime = typeof process === 'object' || typeof Deno === 'object' && typeof Bun === 'object'; +const isWebWorker = !isSystemRuntime && typeof self === 'object'; + +const intlSegmenter = new Intl.Segmenter(); + +for (const [title, input] of testcases) { + group(title, () => { + summary(() => { + barplot(() => { + bench('unicode-segmenter (collect)', () => { + do_not_optimize(collectGraphemes(input)); + }).gc('inner').baseline(true); + + bench('unicode-segmenter (split)', () => { + do_not_optimize([...splitGraphemes(input)]); + }).gc('inner'); + + bench('Intl.Segmenter (naive)', () => { + do_not_optimize([...intlSegmenter.segment(input)].map(s => s.segment)); + }).gc('inner'); + }); + }); + }); +} + +await run({ + format: (!isWebWorker && process.env.MITATA_FORMAT) || 'mitata', + ...isWebWorker && { + colors: false, + print(s) { + self.postMessage({ type: 'log', message: s }); + }, + }, + ...isSystemRuntime && { + colors: !process.env.NO_COLOR, + }, +}); + +if (isWebWorker) { + self.postMessage({ type: 'done' }); +} diff --git a/package.json b/package.json index aad4be9..13054e6 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,8 @@ "perf:grapheme:browser": "vite -c benchmark/grapheme/vite.config.js", "perf:grapheme:hermes": "node benchmark/grapheme/perf-hermes.js", "perf:grapheme:quickjs": "node benchmark/grapheme/perf-quickjs.js", - "perf:grapheme-count": "node --expose-gc benchmark/grapheme/perf-count.js" + "perf:grapheme-count": "node --expose-gc benchmark/grapheme/perf-count.js", + "perf:grapheme-collect": "node --expose-gc benchmark/grapheme/perf-collect.js" }, "alias": { "process": false diff --git a/src/grapheme.js b/src/grapheme.js index e26b776..bf73724 100644 --- a/src/grapheme.js +++ b/src/grapheme.js @@ -359,6 +359,66 @@ export function* splitGraphemes(input) { for (let s of graphemeSegments(input)) yield s.segment; } +/** + * Collect all extended grapheme clusters in given text. + * + * This is a faster alternative to {@link splitGraphemes}, as it packs sliced segments directly into the result array. + * However, for large inputs, using the {@link splitGraphemes} is more memory-efficient. + * + * @param {string} input + * @return {string[]} array of grapheme clusters + * + * @see {@link splitGraphemes} for a large text input. + * + * @example + * collectGraphemes('abc') // => ['a', 'b', 'c'] + */ +export function collectGraphemes(input) { + /** @type {string[]} */ + let result = []; + + let len = input.length; + if (len === 0) return result; + + let cp = /** @type {number} */ (input.codePointAt(0)); + let cursor = cp > BMP_MAX ? 2 : 1; + + /** Category of the last consumed code point */ + let catBefore = cat(cp); + + /** Packed sequence state */ + let st = (0xC418 >> catBefore) & 1 ? nextState(0, catBefore, cp) : 0; + + /** Start index of the current segment */ + let index = 0; + + while (cursor < len) { + cp = /** @type {number} */ (input.codePointAt(cursor)); + let catAfter = cat(cp); + let d = PAIR[catBefore << 4 | catAfter]; + let boundary; + if (d === 0) boundary = true; + else if (d === 1) boundary = false; + else if (d === 2) boundary = !(st & 1); + else if (d === 3) boundary = !(st & 4); + else boundary = (st & 24) !== 16; + + st = (0xC418 >> catAfter) & 1 && (st !== 0 || catAfter !== 3) + ? nextState(st, catAfter, cp) + : 0; + + if (boundary) { + result.push(input.slice(index, cursor)); + index = cursor; + } + cursor += cp > BMP_MAX ? 2 : 1; + catBefore = catAfter; + } + + result.push(input.slice(index)); + return result; +} + // Keep one live segmenter and its result reachable so their hidden classes // stay strongly referenced. Otherwise a major GC while no segmenter is // alive clears the maps embedded weakly in JIT-optimized code, and the diff --git a/test/grapheme.js b/test/grapheme.js index 5bac9ec..8d9056f 100644 --- a/test/grapheme.js +++ b/test/grapheme.js @@ -7,8 +7,9 @@ import fc from 'fast-check'; import { GraphemeCategory, graphemeSegments, - splitGraphemes, countGraphemes, + splitGraphemes, + collectGraphemes, } from 'unicode-segmenter/grapheme'; import { assertObjectContaining } from './_helper.js'; @@ -105,7 +106,7 @@ test('countGraphemes', async t => { }); }); -test('splitGrapheme', async t => { +test('splitGraphemes', async t => { await t.test('latin', () => { assert.deepEqual( [...splitGraphemes('abcd')], @@ -160,6 +161,63 @@ test('splitGrapheme', async t => { }); }); +test('collectGraphemes', async t => { + await t.test('latin', () => { + assert.deepEqual( + collectGraphemes('abcd'), + ['a', 'b', 'c', 'd'], + ); + }); + + await t.test('flags', () => { + assert.deepEqual( + collectGraphemes('🇷🇸🇮🇴'), + ['🇷🇸', '🇮🇴'], + ); + }); + + await t.test('emoji', () => { + assert.deepEqual( + collectGraphemes('👻👩‍👩‍👦‍👦'), + ['👻', '👩‍👩‍👦‍👦'], + ); + assert.deepEqual( + collectGraphemes('🌷🎁💩😜👍🏳️‍🌈'), + ['🌷', '🎁', '💩', '😜', '👍', '🏳️‍🌈'], + ); + }); + + await t.test('diacritics as combining marks', () => { + assert.deepEqual( + collectGraphemes('Ĺo͂řȩm̅'), + ['Ĺ', 'o͂', 'ř', 'ȩ', 'm̅'], + ); + }); + + await t.test('Jamo', () => { + assert.deepEqual( + collectGraphemes('가갉'), + ['가', '갉'], + ); + }); + + await t.test('Hindi', () => { + assert.deepEqual( + collectGraphemes('अनुच्छेद'), + ['अ', 'नु', 'च्छे', 'द'], + ); + }); + + await t.test('demonic', () => { + assert.deepEqual( + collectGraphemes('Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'), + ['Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍', 'A̴̵̜̰͔ͫ͗͢', 'L̠ͨͧͩ͘', 'G̴̻͈͍͔̹̑͗̎̅͛́', 'Ǫ̵̹̻̝̳͂̌̌͘', '!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'], + ); + }); +}); + + + test('conformance', async t => { fc.configureGlobal({ // Fix seed here for stable coverage report @@ -175,6 +233,7 @@ test('conformance', async t => { assertObjectContaining([...graphemeSegments(input)], expected); assert.equal(countGraphemes(input), expected.length); assert.deepEqual([...splitGraphemes(input)], expected.map(s => s.segment)); + assert.deepEqual(collectGraphemes(input), expected.map(s => s.segment)); } await t.test('any string', () => {