Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/heavy-grapes-accept.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
"unicode-segmenter": patch
---

Add `collectGraphemes(input)` API that collects grapheme clusters into an array directly.

This is a fast version of `[...splitGraphemes(input)]`, which is acually the most used pattern in practice.

It's 2-4x faster than the iterator-based approach. However, it collects all grapheme clusters at once, so it's not good for large text or streamed input.
24 changes: 19 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ Utilities for text segmentation by extended grapheme cluster rules.

You can retrieve all grapheme segments using the `graphemeSegments()` generator.

Each segment is returned as a `GraphemeSegment` object containing the `segment` (the substring), the starting `index`, and the full `input` string. This pattern is similar to the [`Intl.Segmenter`] API, so it can be a drop-in replacement.
Each segment is returned as a `GraphemeSegmentOutput` object containing the `segment` (the substring), the starting `index`, and the full `input` string. This pattern is similar to the [`Intl.Segmenter`] API, so it can be a drop-in replacement.

```js
import { graphemeSegments } from 'unicode-segmenter/grapheme';
Expand Down Expand Up @@ -81,6 +81,20 @@ import { splitGraphemes } from 'unicode-segmenter/grapheme';
// 4: 2️⃣
```

#### Example: Collect graphemes into an array

If you need the result as an array, use `collectGraphemes()` for convenience.

This is a fast version of `[...splitGraphemes(str)]`, 2-4x faster.

```js
import { collectGraphemes } from 'unicode-segmenter/grapheme';

collectGraphemes('#️⃣*️⃣0️⃣1️⃣2️⃣'); // => ["#️⃣", "*️⃣", "0️⃣", "1️⃣", "2️⃣"]
```

However, it collects all grapheme clusters at once, so it's not good for large text or streamed input.

#### Example: Count graphemes

If you need only the count, use `countGraphemes()`, which is optimized to avoid allocations for segments.
Expand Down Expand Up @@ -223,8 +237,8 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb

| Name | Unicode® | ESM? | Size | Size (min) | Size (min+gzip) | Size (min+br) | Size (min+zstd) |
|--------------------------------------|----------|------|--------:|-----------:|----------------:|--------------:|----------------:|
| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,202 | 2,507 |
| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 10,092 | 5,573 | 2,681 | 2,402 | 2,722 |
| `unicode-segmenter/grapheme` | 17.0.0 | ✔️ | 8,782 | 5,069 | 2,462 | 2,218 | 2,508 |
| `unicode-segmenter/grapheme` (full*) | 17.0.0 | ✔️ | 11,182 | 5,927 | 2,820 | 2,492 | 2,865 |
| `graphemer` | 15.0.0 | ✖️ | 410,435 | 95,104 | 15,752 | 10,660 | 15,911 |
| `grapheme-splitter` | 10.0.0 | ✖️ | 122,254 | 23,682 | 7,852 | 4,802 | 6,753 |
| `@formatjs/intl-segmenter`* | 17.0.0 | ✖️ | 268,301 | 176,759 | 45,988 | 31,701 | 45,370 |
Expand All @@ -241,8 +255,8 @@ Since [Hermes doesn't support the `Intl.Segmenter` API](https://github.com/faceb

| Name | Bytecode size | Bytecode size (gzip)* |
|-------------------------------------|--------------:|----------------------:|
| `unicode-segmenter/grapheme` | 19,538 | 10,957 |
| `unicode-segmenter/grapheme` (full) | 19,743 | 11,080 |
| `unicode-segmenter/grapheme` | 20,203 | 11,224 |
| `unicode-segmenter/grapheme` (full) | 20,408 | 11,356 |
| `graphemer` | 134,085 | 31,770 |
| `grapheme-splitter` | 63,942 | 19,165 |
| `@formatjs/intl-segmenter` | 329,547 | 136,751 |
Expand Down
13 changes: 13 additions & 0 deletions benchmark/grapheme/codspeed.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
graphemeSegments,
countGraphemes,
splitGraphemes,
collectGraphemes,
} from 'unicode-segmenter/grapheme';
import { testcases } from './_testcases.js';

Expand Down Expand Up @@ -50,5 +51,17 @@ for (const [name, input] of testcases) {
});
}

for (const [name, input] of testcases) {
bench.add(`collectGraphemes - ${name}`, () => {
void collectGraphemes(input);
}, {
beforeAll() {
for (let i = 0; i < 2000; i++) {
void collectGraphemes(input);
}
},
});
}

await bench.run();
console.table(bench.table());
57 changes: 57 additions & 0 deletions benchmark/grapheme/perf-collect.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import {
group,
summary,
barplot,
bench,
run,
do_not_optimize,
} from 'mitata';

import {
splitGraphemes,
collectGraphemes,
} from '../../src/grapheme.js';
import { testcases } from './_testcases.js';

// Node.js, Deno, Bun
const isSystemRuntime = typeof process === 'object' || typeof Deno === 'object' && typeof Bun === 'object';
const isWebWorker = !isSystemRuntime && typeof self === 'object';

const intlSegmenter = new Intl.Segmenter();

for (const [title, input] of testcases) {
group(title, () => {
summary(() => {
barplot(() => {
bench('unicode-segmenter (collect)', () => {
do_not_optimize(collectGraphemes(input));
}).gc('inner').baseline(true);

bench('unicode-segmenter (split)', () => {
do_not_optimize([...splitGraphemes(input)]);
}).gc('inner');

bench('Intl.Segmenter (naive)', () => {
do_not_optimize([...intlSegmenter.segment(input)].map(s => s.segment));
}).gc('inner');
});
});
});
}

await run({
format: (!isWebWorker && process.env.MITATA_FORMAT) || 'mitata',
...isWebWorker && {
colors: false,
print(s) {
self.postMessage({ type: 'log', message: s });
},
},
...isSystemRuntime && {
colors: !process.env.NO_COLOR,
},
});

if (isWebWorker) {
self.postMessage({ type: 'done' });
}
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@
"perf:grapheme:browser": "vite -c benchmark/grapheme/vite.config.js",
"perf:grapheme:hermes": "node benchmark/grapheme/perf-hermes.js",
"perf:grapheme:quickjs": "node benchmark/grapheme/perf-quickjs.js",
"perf:grapheme-count": "node --expose-gc benchmark/grapheme/perf-count.js"
"perf:grapheme-count": "node --expose-gc benchmark/grapheme/perf-count.js",
"perf:grapheme-collect": "node --expose-gc benchmark/grapheme/perf-collect.js"
},
"alias": {
"process": false
Expand Down
60 changes: 60 additions & 0 deletions src/grapheme.js
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,66 @@ export function* splitGraphemes(input) {
for (let s of graphemeSegments(input)) yield s.segment;
}

/**
* Collect all extended grapheme clusters in given text.
*
* This is a faster alternative to {@link splitGraphemes}, as it packs sliced segments directly into the result array.
* However, for large inputs, using the {@link splitGraphemes} is more memory-efficient.
*
* @param {string} input
* @return {string[]} array of grapheme clusters
*
* @see {@link splitGraphemes} for a large text input.
*
* @example
* collectGraphemes('abc') // => ['a', 'b', 'c']
*/
export function collectGraphemes(input) {
/** @type {string[]} */
let result = [];

let len = input.length;
if (len === 0) return result;

let cp = /** @type {number} */ (input.codePointAt(0));
let cursor = cp > BMP_MAX ? 2 : 1;

/** Category of the last consumed code point */
let catBefore = cat(cp);

/** Packed sequence state */
let st = (0xC418 >> catBefore) & 1 ? nextState(0, catBefore, cp) : 0;

/** Start index of the current segment */
let index = 0;

while (cursor < len) {
cp = /** @type {number} */ (input.codePointAt(cursor));
let catAfter = cat(cp);
let d = PAIR[catBefore << 4 | catAfter];
let boundary;
if (d === 0) boundary = true;
else if (d === 1) boundary = false;
else if (d === 2) boundary = !(st & 1);
else if (d === 3) boundary = !(st & 4);
else boundary = (st & 24) !== 16;

st = (0xC418 >> catAfter) & 1 && (st !== 0 || catAfter !== 3)
? nextState(st, catAfter, cp)
: 0;

if (boundary) {
result.push(input.slice(index, cursor));
index = cursor;
}
cursor += cp > BMP_MAX ? 2 : 1;
catBefore = catAfter;
}

result.push(input.slice(index));
return result;
}

// Keep one live segmenter and its result reachable so their hidden classes
// stay strongly referenced. Otherwise a major GC while no segmenter is
// alive clears the maps embedded weakly in JIT-optimized code, and the
Expand Down
63 changes: 61 additions & 2 deletions test/grapheme.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import fc from 'fast-check';
import {
GraphemeCategory,
graphemeSegments,
splitGraphemes,
countGraphemes,
splitGraphemes,
collectGraphemes,
} from 'unicode-segmenter/grapheme';
import { assertObjectContaining } from './_helper.js';

Expand Down Expand Up @@ -105,7 +106,7 @@ test('countGraphemes', async t => {
});
});

test('splitGrapheme', async t => {
test('splitGraphemes', async t => {
await t.test('latin', () => {
assert.deepEqual(
[...splitGraphemes('abcd')],
Expand Down Expand Up @@ -160,6 +161,63 @@ test('splitGrapheme', async t => {
});
});

test('collectGraphemes', async t => {
await t.test('latin', () => {
assert.deepEqual(
collectGraphemes('abcd'),
['a', 'b', 'c', 'd'],
);
});

await t.test('flags', () => {
assert.deepEqual(
collectGraphemes('🇷🇸🇮🇴'),
['🇷🇸', '🇮🇴'],
);
});

await t.test('emoji', () => {
assert.deepEqual(
collectGraphemes('👻👩‍👩‍👦‍👦'),
['👻', '👩‍👩‍👦‍👦'],
);
assert.deepEqual(
collectGraphemes('🌷🎁💩😜👍🏳️‍🌈'),
['🌷', '🎁', '💩', '😜', '👍', '🏳️‍🌈'],
);
});

await t.test('diacritics as combining marks', () => {
assert.deepEqual(
collectGraphemes('Ĺo͂řȩm̅'),
['Ĺ', 'o͂', 'ř', 'ȩ', 'm̅'],
);
});

await t.test('Jamo', () => {
assert.deepEqual(
collectGraphemes('가갉'),
['가', '갉'],
);
});

await t.test('Hindi', () => {
assert.deepEqual(
collectGraphemes('अनुच्छेद'),
['अ', 'नु', 'च्छे', 'द'],
);
});

await t.test('demonic', () => {
assert.deepEqual(
collectGraphemes('Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍A̴̵̜̰͔ͫ͗͢L̠ͨͧͩ͘G̴̻͈͍͔̹̑͗̎̅͛́Ǫ̵̹̻̝̳͂̌̌͘!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'),
['Z͑ͫ̓ͪ̂ͫ̽͏̴̙̤̞͉͚̯̞̠͍', 'A̴̵̜̰͔ͫ͗͢', 'L̠ͨͧͩ͘', 'G̴̻͈͍͔̹̑͗̎̅͛́', 'Ǫ̵̹̻̝̳͂̌̌͘', '!͖̬̰̙̗̿̋ͥͥ̂ͣ̐́́͜͞'],
);
});
});



test('conformance', async t => {
fc.configureGlobal({
// Fix seed here for stable coverage report
Expand All @@ -175,6 +233,7 @@ test('conformance', async t => {
assertObjectContaining([...graphemeSegments(input)], expected);
assert.equal(countGraphemes(input), expected.length);
assert.deepEqual([...splitGraphemes(input)], expected.map(s => s.segment));
assert.deepEqual(collectGraphemes(input), expected.map(s => s.segment));
}

await t.test('any string', () => {
Expand Down
Loading