Skip to content

Commit 41a6892

Browse files
neueccneuecc
committed
v2, adopt SerializerFoundation
1 parent 7dd3942 commit 41a6892

23 files changed

Lines changed: 700 additions & 547 deletions

‎README.md‎

Lines changed: 60 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,11 @@ In this case, when the `CodeDiagnostic[]` count is large, there is a significant
137137

138138
## ToonEncoder
139139

140+
It adopts [SerializerFoundation](https://github.com/Cysharp/SerializerFoundation) as the foundation for the Serializer. `IWriteBuffer` is SerializerFoundation's type.
141+
140142
### JsonElement to Toon
141143

142-
`ToonEncoder.Encode` supports conversion from `JsonElement` to `string` or `byte[]`, and writing to `IBufferWriter<byte>` or `ToonWriter`.
144+
`ToonEncoder.Encode` supports conversion from `JsonElement` to `string` or `byte[]`, and writing to `IWriteBuffer`, `IBufferWriter<byte>` or `ToonWriter`.
143145

144146
```csharp
145147
namespace Cysharp.AI;
@@ -148,19 +150,22 @@ public static class ToonEncoder
148150
{
149151
public static string Encode(JsonElement element);
150152

151-
public static void Encode<TBufferWriter>(ref TBufferWriter bufferWriter, JsonElement element)
152-
where TBufferWriter : IBufferWriter<byte>;
153+
public static void Encode<TBufferWriter>(TBufferWriter bufferWriter, JsonElement element)
154+
where TBufferWriter : class, IBufferWriter<byte>;
155+
156+
public static void Encode<TWriteBuffer>(ref TWriteBuffer writeBuffer, JsonElement element)
157+
where TWriteBuffer : struct, IWriteBuffer;
153158

154-
public static void Encode<TBufferWriter>(ref ToonWriter<TBufferWriter> toonWriter, JsonElement element)
155-
where TBufferWriter : IBufferWriter<byte>;
159+
public static void Encode<TWriteBuffer>(ref ToonWriter<TWriteBuffer> toonWriter, JsonElement element)
160+
where TWriteBuffer : struct, IWriteBuffer;
156161

157162
public static byte[] EncodeToUtf8Bytes(JsonElement element);
158163

159164
public static async ValueTask EncodeAsync(Stream utf8Stream, JsonElement element, CancellationToken cancellationToken = default);
160165
}
161166
```
162167

163-
Using the `IBufferWriter<byte>` overload writes data directly in UTF-8, which yields better performance than going through `string` conversion.
168+
Using the `IWriteBuffer` or `IBufferWriter<byte>` overload writes data directly in UTF-8, which yields better performance than going through `string` conversion.
164169

165170
If the `JsonElement` is an `array` where all elements appear in the same order and all are primitives (not Array or Object), the `EncodeAsTabularArray` method provides higher performance conversion.
166171

@@ -171,15 +176,18 @@ public static class ToonEncoder
171176
{
172177
public static string EncodeAsTabularArray(JsonElement array);
173178

174-
public static void EncodeAsTabularArray<TBufferWriter>(ref TBufferWriter bufferWriter, JsonElement array)
175-
where TBufferWriter : IBufferWriter<byte>;
179+
public static void EncodeAsTabularArray<TBufferWriter>(TBufferWriter bufferWriter, JsonElement array)
180+
where TBufferWriter : class, IBufferWriter<byte>;
181+
182+
public static void EncodeAsTabularArray<TWriteBuffer>(ref TWriteBuffer writeBuffer, JsonElement array)
183+
where TWriteBuffer : struct, IWriteBuffer
176184

177185
public static byte[] EncodeAsTabularArrayToUtf8Bytes(JsonElement array);
178186

179187
public static async ValueTask EncodeAsTabularArrayAsync(Stream utf8Stream, JsonElement array, CancellationToken cancellationToken = default);
180188

181-
public static void EncodeAsTabularArray<TBufferWriter>(ref ToonWriter<TBufferWriter> toonWriter, JsonElement array)
182-
where TBufferWriter : IBufferWriter<byte>;
189+
public static void EncodeAsTabularArray<TWriteBuffer>(ref ToonWriter<TWriteBuffer> toonWriter, JsonElement array)
190+
where TWriteBuffer : struct, IWriteBuffer
183191
}
184192
```
185193

@@ -194,37 +202,43 @@ public static class ToonEncoder
194202
{
195203
public static string Encode<T>(T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default);
196204

197-
public static void Encode<TBufferWriter, T>(ref TBufferWriter bufferWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
198-
where TBufferWriter : IBufferWriter<byte>;
205+
public static void Encode<TBufferWriter, T>(TBufferWriter bufferWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
206+
where TBufferWriter : class, IBufferWriter<byte>;
207+
208+
public static void Encode<TWriteBuffer, T>(ref TWriteBuffer writeBuffer, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
209+
where TWriteBuffer : struct, IWriteBuffer;
199210

200211
public static byte[] EncodeToUtf8Bytes<T>(T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default);
201212

202213
public static async ValueTask EncodeAsync<T>(Stream utf8Stream, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default, CancellationToken cancellationToken = default);
203214

204-
public static void Encode<TBufferWriter, T>(ref ToonWriter<TBufferWriter> toonWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
205-
where TBufferWriter : IBufferWriter<byte>;
215+
public static void Encode<TWriteBuffer, T>(ref ToonWriter<TWriteBuffer> toonWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
216+
where TWriteBuffer : struct, IWriteBuffer;
206217

207218
// AsTabularArray
208219
209220
public static string EncodeAsTabularArray<T>(T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default);
210221

211-
public static void EncodeAsTabularArray<TBufferWriter, T>(ref TBufferWriter bufferWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
212-
where TBufferWriter : IBufferWriter<byte>;
222+
public static void EncodeAsTabularArray<TBufferWriter, T>(TBufferWriter bufferWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
223+
where TBufferWriter : class, IBufferWriter<byte>;
224+
225+
public static void EncodeAsTabularArray<TWriteBuffer, T>(ref TWriteBuffer writeBuffer, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
226+
where TWriteBuffer : struct, IWriteBuffer;
213227

214228
public static byte[] EncodeAsTabularArrayToUtf8Bytes<T>(T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default);
215229

216230
public static async ValueTask EncodeAsTabularArrayAsync<T>(Stream utf8Stream, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default, CancellationToken cancellationToken = default);
217231

218-
public static void EncodeAsTabularArray<TBufferWriter, T>(ref ToonWriter<TBufferWriter> toonWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
219-
where TBufferWriter : IBufferWriter<byte>;
232+
public static void EncodeAsTabularArray<TWriteBuffer, T>(ref ToonWriter<TWriteBuffer> toonWriter, T value, JsonSerializerOptions? jsonSerializerOptionsWithoutToonConverter = default)
233+
where TWriteBuffer : struct, IWriteBuffer;
220234
}
221235
```
222236

223237
This method accepts `JsonSerializerOptions` as an argument, but do not include `ToonConverter<T>` or `ToonTabularArrayConverter<T>` in its converters (converters generated by `GenerateToonTabularArrayConverter` are acceptable). Doing so will cause recursive serialization resulting in a `StackOverflowException`. When no options are specified, [RecommendJsonSerializerOptions](#recommendjsonserializeroptions) is used by default.
224238

225239
### ToonWriter
226240

227-
ToonWriter is the most primitive API for flexibly controlling output and writing to `IArrayBufferWriter<byte>` in TOON format.
241+
ToonWriter is the most primitive API for flexibly controlling output and writing to `IWriteBuffer` or `IArrayBufferWriter<byte>` in TOON format. When applying it to `IArrayBuffer<byte>`, you need to wrap it in `IWriteBuffer`.
228242

229243
```csharp
230244
using Cysharp.AI;
@@ -233,8 +247,11 @@ using System.Text;
233247

234248
var arrayBufferWriter = new ArrayBufferWriter<byte>();
235249

250+
// wrap to IWriteBuffer
251+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
252+
236253
// create ToonWriter
237-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
254+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
238255

239256
// write as InlineArray
240257
toonWriter.WriteStartInlineArray(5);
@@ -245,8 +262,9 @@ toonWriter.WriteString("date");
245262
toonWriter.WriteString("elderberry");
246263
toonWriter.WriteEndInlineArray();
247264

248-
// Flush writes to the IBufferWriter<byte> passed by ref
249-
toonWriter.Flush();
265+
// Flush writeBuffer.
266+
writeBuffer.Flush();
267+
writeBuffer.Dispose();
250268

251269
// [5]: apple,banana,cherry,date,elderberry
252270
var str = Encoding.UTF8.GetString(arrayBufferWriter.WrittenSpan);
@@ -265,7 +283,8 @@ In object format, wrap with `WriteStartObject`-`WriteEndObject`, and express `ke
265283

266284
```csharp
267285
var arrayBufferWriter = new ArrayBufferWriter<byte>();
268-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
286+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
287+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
269288

270289
// write as object
271290
toonWriter.WriteStartObject();
@@ -278,7 +297,8 @@ toonWriter.WriteNumber(100);
278297

279298
toonWriter.WriteEndObject();
280299

281-
toonWriter.Flush();
300+
writeBuffer.Flush();
301+
writeBuffer.Dispose();
282302

283303
// fruits: apple
284304
// price: 100
@@ -290,7 +310,8 @@ Nested case:
290310

291311
```csharp
292312
var arrayBufferWriter = new ArrayBufferWriter<byte>();
293-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
313+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
314+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
294315

295316
// nested object/array
296317
toonWriter.WriteStartObject();
@@ -307,7 +328,8 @@ toonWriter.WriteEndInlineArray();
307328

308329
toonWriter.WriteEndObject();
309330

310-
toonWriter.Flush();
331+
writeBuffer.Flush();
332+
writeBuffer.Dispose();
311333

312334
// grade: first
313335
// ids[3]: 1,5,9
@@ -327,7 +349,8 @@ One-dimensional arrays of primitives can be expressed as InlineArray. Start with
327349

328350
```csharp
329351
var arrayBufferWriter = new ArrayBufferWriter<byte>();
330-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
352+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
353+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
331354

332355
// write as inline-array
333356
toonWriter.WriteStartInlineArray(3);
@@ -336,7 +359,8 @@ toonWriter.WriteNumber(20);
336359
toonWriter.WriteNumber(30);
337360
toonWriter.WriteEndInlineArray();
338361

339-
toonWriter.Flush();
362+
writeBuffer.Flush();
363+
writeBuffer.Dispose();
340364

341365
// [3] 10,20,30
342366
var str = Encoding.UTF8.GetString(arrayBufferWriter.WrittenSpan);
@@ -349,7 +373,8 @@ The CSV-like tabular format characteristic of TOON can be expressed as TabularAr
349373

350374
```csharp
351375
var arrayBufferWriter = new ArrayBufferWriter<byte>();
352-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
376+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
377+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
353378

354379
// write as Tabular Array
355380
toonWriter.WriteStartTabularArray(2, ["id", "name", "role"]);
@@ -366,7 +391,8 @@ toonWriter.WriteString("user");
366391

367392
toonWriter.WriteEndTabularArray();
368393

369-
toonWriter.Flush();
394+
writeBuffer.Flush();
395+
writeBuffer.Dispose();
370396

371397
// [2]{id,name,role}:
372398
// 1,Alice,admin
@@ -381,7 +407,8 @@ Arrays containing non-primitive elements (objects, arrays, etc.) start with `Wri
381407

382408
```csharp
383409
var arrayBufferWriter = new ArrayBufferWriter<byte>();
384-
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
410+
var writeBuffer = new NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>(arrayBufferWriter);
411+
var toonWriter = new ToonWriter<NonRefBufferWriterWriteBuffer<ArrayBufferWriter<byte>>>(ref writeBuffer);
385412

386413
// write as NonUniform Array
387414
toonWriter.WriteStartNonUniformArray(2);
@@ -394,7 +421,8 @@ toonWriter.WriteString("Alice");
394421

395422
toonWriter.WriteEndNonUniformArray();
396423

397-
toonWriter.Flush();
424+
writeBuffer.Flush();
425+
writeBuffer.Dispose();
398426

399427
// [2]:
400428
// - 1

‎sandbox/BenchmarkSuite/BenchmarkSuite.csproj‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,38 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

3-
<PropertyGroup>
4-
<TargetFramework>net10.0</TargetFramework>
5-
<LangVersion>14</LangVersion>
6-
<OutputType>Exe</OutputType>
7-
<Nullable>enable</Nullable>
3+
<PropertyGroup>
4+
<TargetFramework>net10.0</TargetFramework>
5+
<LangVersion>14</LangVersion>
6+
<OutputType>Exe</OutputType>
7+
<Nullable>enable</Nullable>
8+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
9+
<NoWarn>1701;1702;CS0436</NoWarn>
10+
</PropertyGroup>
811

9-
</PropertyGroup>
12+
<ItemGroup>
13+
<PackageReference Include="BenchmarkDotNet" Version="0.15.2" />
14+
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="18.0.36313.1" />
15+
<PackageReference Include="Toon" Version="0.0.1-preview-01" />
16+
<PackageReference Include="ToonSharp" Version="1.0.0" />
17+
<PackageReference Include="ToonNet" Version="1.0.4" />
18+
<PackageReference Include="Toon.DotNet" Version="1.5.1" />
19+
</ItemGroup>
1020

11-
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
12-
<NoWarn>1701;1702;CS0436</NoWarn>
13-
</PropertyGroup>
21+
<Choose>
22+
<When Condition="'$(Configuration)'=='NuGetVersion'">
23+
<ItemGroup>
24+
<PackageReference Include="ToonEncoder" Version="1.0.2" />
25+
</ItemGroup>
26+
</When>
27+
<Otherwise>
28+
<ItemGroup>
29+
<ProjectReference Include="..\..\src\ToonEncoder\ToonEncoder.csproj" />
30+
<ProjectReference Include="..\..\src\ToonEncoder.Generator\ToonEncoder.Generator.csproj">
31+
<OutputItemType>Analyzer</OutputItemType>
32+
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
33+
</ProjectReference>
34+
</ItemGroup>
35+
</Otherwise>
36+
</Choose>
1437

15-
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
16-
<NoWarn>1701;1702;CS0436</NoWarn>
17-
</PropertyGroup>
18-
19-
<ItemGroup>
20-
<PackageReference Include="BenchmarkDotNet" Version="0.15.2" />
21-
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="18.0.36313.1" />
22-
<PackageReference Include="Toon" Version="0.0.1-preview-01" />
23-
<PackageReference Include="ToonSharp" Version="1.0.0" />
24-
<PackageReference Include="ToonNet" Version="1.0.4" />
25-
<PackageReference Include="Toon.DotNet" Version="1.5.1" />
26-
</ItemGroup>
27-
28-
<ItemGroup>
29-
<ProjectReference Include="..\..\src\ToonEncoder\ToonEncoder.csproj" />
30-
<ProjectReference Include="..\..\src\ToonEncoder.Generator\ToonEncoder.Generator.csproj">
31-
<OutputItemType>Analyzer</OutputItemType>
32-
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
33-
</ProjectReference>
34-
</ItemGroup>
3538
</Project>

‎sandbox/BenchmarkSuite/EncodeBenchmark.cs‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ namespace BenchmarkSuite
1717
[GenerateToonTabularArrayConverter]
1818
public record Person(int Id, string Name, int Age);
1919

20-
[Orderer(SummaryOrderPolicy.FastestToSlowest)]
20+
[Orderer(SummaryOrderPolicy.Default)]
2121
public class EncodeBenchmark
2222
{
2323
Person[] data;
@@ -49,39 +49,39 @@ public string ToonEncoder_EncodeAsTabularArray_SourceGenerated()
4949
return Cysharp.AI.Converters.BenchmarkSuite_PersonTabularArrayConverter.EncodeAsTabularArray(data);
5050
}
5151

52-
// https://github.com/StefH/Toon.NET
53-
[Benchmark]
54-
public string Toon_Encode()
55-
{
56-
return global::Toon.ToonEncoder.Encode(data);
57-
}
52+
//// https://github.com/StefH/Toon.NET
53+
//[Benchmark]
54+
//public string Toon_Encode()
55+
//{
56+
// return global::Toon.ToonEncoder.Encode(data);
57+
//}
5858

59-
// https://github.com/0xZunia/ToonSharp
60-
[Benchmark]
61-
public string ToonSharp_Serialize()
62-
{
63-
return global::ToonSharp.ToonSerializer.Serialize(data);
64-
}
59+
//// https://github.com/0xZunia/ToonSharp
60+
//[Benchmark]
61+
//public string ToonSharp_Serialize()
62+
//{
63+
// return global::ToonSharp.ToonSerializer.Serialize(data);
64+
//}
6565

66-
// https://github.com/Nicola898989/ToonNet
67-
[Benchmark]
68-
public string ToonNet_Encode()
69-
{
70-
return global::ToonNetSerializer.ToonNet.Encode(data);
71-
}
66+
//// https://github.com/Nicola898989/ToonNet
67+
//[Benchmark]
68+
//public string ToonNet_Encode()
69+
//{
70+
// return global::ToonNetSerializer.ToonNet.Encode(data);
71+
//}
7272

73-
// https://github.com/CharlesHunt/ToonDotNet
74-
[Benchmark]
75-
public string ToonDotNet_Encode()
76-
{
77-
return global::ToonFormat.Toon.Encode(data);
78-
}
73+
//// https://github.com/CharlesHunt/ToonDotNet
74+
//[Benchmark]
75+
//public string ToonDotNet_Encode()
76+
//{
77+
// return global::ToonFormat.Toon.Encode(data);
78+
//}
7979

80-
// Official impl: https://github.com/toon-format/toon-dotnet
81-
[Benchmark]
82-
public string ToonOfficial_Encode()
83-
{
84-
return global::Toon.Format.ToonEncoder.Encode(data);
85-
}
80+
//// Official impl: https://github.com/toon-format/toon-dotnet
81+
//[Benchmark]
82+
//public string ToonOfficial_Encode()
83+
//{
84+
// return global::Toon.Format.ToonEncoder.Encode(data);
85+
//}
8686
}
8787
}

0 commit comments

Comments
 (0)