Skip to content

Commit 0c07da3

Browse files
popemattpopematt
authored
Adds Ion 1.1 raw text writer implementation (#1123)
1 parent b03d7ca commit 0c07da3

9 files changed

Lines changed: 1910 additions & 69 deletions

File tree

‎src/main/java/com/amazon/ion/bytecode/bin11/OpCode.kt‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,18 +95,21 @@ internal object OpCode {
9595
const val STRUCT_LENGTH_15 = 0xDF
9696

9797
const val IVM = 0xE0
98+
const val NO_ARGUMENT = 0xE0
99+
98100
const val DIRECTIVE_SET_SYMBOLS = 0xE1
99101
const val DIRECTIVE_ADD_SYMBOLS = 0xE2
100102
const val DIRECTIVE_SET_MACROS = 0xE3
101103
const val DIRECTIVE_ADD_MACROS = 0xE4
102104
const val DIRECTIVE_USE = 0xE5
103105
const val DIRECTIVE_MODULE = 0xE6
104-
const val DIRECTIVE_ENCODING = 0xE7
105-
const val TAGGED_PLACEHOLDER = 0xE8
106-
const val TAGGED_PLACEHOLDER_WITH_DEFAULT = 0xE9
107-
const val TAGLESS_PLACEHOLDER = 0xEA
106+
const val DIRECTIVE_IMPORT = 0xE7
107+
const val DIRECTIVE_ENCODING = 0xE8
108+
109+
const val TAGGED_PLACEHOLDER = 0xE9
110+
const val TAGGED_PLACEHOLDER_WITH_DEFAULT = 0xEA
111+
const val TAGLESS_PLACEHOLDER = 0xEB
108112

109-
const val NO_ARGUMENT = 0xEB
110113
const val NOP = 0xEC
111114
const val NOP_L = 0xED
112115

‎src/main/java/com/amazon/ion/impl/BufferedOutputStreamFastAppendable.kt‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package com.amazon.ion.impl
4+
5+
import com.amazon.ion.impl.bin.Block
6+
import com.amazon.ion.impl.bin.BlockAllocator
7+
import com.amazon.ion.util._Private_FastAppendable
8+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
9+
import java.io.OutputStream
10+
11+
/**
12+
* A [_Private_FastAppendable] that buffers data to blocks of memory which are managed by a [BlockAllocator]. Only when
13+
* [flush] is called are the blocks written to the wrapped [OutputStream].
14+
*
15+
* This is necessary for cases where an [IonManagedWriter_1_1] over Ion text needs to emit encoding directives that are
16+
* not known in advance. The [OutputStreamFastAppendable] class only buffers a fixed amount of data, so it is not safe
17+
* to use if there are system values to be written. For a sufficiently large user value, an [OutputStreamFastAppendable]
18+
* can end up flushing partial or whole user values flushing to the [OutputStream] before the [IonManagedWriter_1_1] can
19+
* write the system value that it depends on.
20+
*
21+
* Once [IonManagedWriter_1_1] supports an auto-flush feature, then this class will have very little practical
22+
* difference from [OutputStreamFastAppendable] for the case where no system values are needed.
23+
*
24+
* TODO:
25+
* - Add proper tests
26+
*
27+
* @see BufferedAppendableFastAppendable
28+
* @see OutputStreamFastAppendable
29+
*/
30+
internal class BufferedOutputStreamFastAppendable(
31+
@SuppressFBWarnings("EI_EXPOSE_REP2", justification = "We're intentionally storing a reference to a mutable object because we need to write to it.")
32+
private val out: OutputStream,
33+
private val allocator: BlockAllocator,
34+
/**
35+
* The minimum utilization of a block before a longer value
36+
* can skip the end of a block and just start a new block.
37+
*/
38+
minBlockUtilization: Double = 1.0,
39+
) : OutputStream(), _Private_FastAppendable {
40+
41+
init {
42+
// 0.0 would have the possibility of wasting entire blocks.
43+
// 0.5 is somewhat arbitrary, but at least sensible that you should use at least
44+
// half of a block before moving on to the next block.
45+
require(minBlockUtilization in 0.5..1.0) { "minBlockUtilization must be between 0.5 and 1" }
46+
require(allocator.blockSize > 10)
47+
}
48+
49+
private val maxBlockWaste: Int = (allocator.blockSize * (1.0 - minBlockUtilization)).toInt()
50+
51+
private var index = -1
52+
private val blocks = mutableListOf<Block>()
53+
private var current: Block = nextBlock()
54+
55+
private fun nextBlock(): Block {
56+
index++
57+
if (index < 0) throw IllegalStateException("This output stream is closed.")
58+
if (index >= blocks.size) blocks.add(allocator.allocateBlock())
59+
return blocks[index]
60+
}
61+
62+
override fun close() {
63+
try {
64+
flush()
65+
} finally {
66+
blocks.onEach { it.close() }.clear()
67+
index = Int.MIN_VALUE
68+
}
69+
}
70+
71+
override fun flush() {
72+
blocks.forEach { block ->
73+
out.write(block.data, 0, block.limit)
74+
block.reset()
75+
}
76+
index = 0
77+
current = blocks[index]
78+
out.flush()
79+
}
80+
81+
override fun write(b: Int) {
82+
if (current.remaining() < 1) current = nextBlock()
83+
val block = current
84+
block.data[block.limit] = b.toByte()
85+
block.limit++
86+
}
87+
88+
override fun write(b: ByteArray, off: Int, len: Int) {
89+
if (len > current.remaining()) {
90+
if (current.remaining() < maxBlockWaste && len < allocator.blockSize) {
91+
current = nextBlock()
92+
} else {
93+
writeBytesSlow(b, off, len)
94+
return
95+
}
96+
}
97+
val block = current
98+
System.arraycopy(b, off, block.data, block.limit, len)
99+
block.limit += len
100+
}
101+
102+
// slow in the sense that we do all kind of block boundary checking
103+
private fun writeBytesSlow(bytes: ByteArray, _off: Int, _len: Int) {
104+
var off = _off
105+
var len = _len
106+
while (len > 0) {
107+
val block = current
108+
val amount = Math.min(len, block.remaining())
109+
System.arraycopy(bytes, off, block.data, block.limit, amount)
110+
block.limit += amount
111+
off += amount
112+
len -= amount
113+
if (block.remaining() == 0) {
114+
current = nextBlock()
115+
}
116+
}
117+
}
118+
119+
override fun append(c: Char): Appendable = apply { if (c.code < 0x80) appendAscii(c) else appendUtf16(c) }
120+
121+
override fun append(csq: CharSequence): Appendable = apply { append(csq, 0, csq.length) }
122+
123+
override fun append(csq: CharSequence, start: Int, end: Int): Appendable {
124+
for (i in start until end) {
125+
append(csq[i])
126+
}
127+
return this
128+
}
129+
130+
override fun appendAscii(c: Char) {
131+
assert(c.code < 0x80)
132+
write(c.code)
133+
}
134+
135+
override fun appendAscii(csq: CharSequence) = appendAscii(csq, 0, csq.length)
136+
137+
override fun appendAscii(csq: CharSequence, start: Int, end: Int) {
138+
if (csq is String) {
139+
var pos = start
140+
val len = end - start
141+
if (len > current.remaining() && current.remaining() < maxBlockWaste && len < allocator.blockSize) {
142+
current = nextBlock()
143+
}
144+
while (true) {
145+
val copyAmount = minOf(current.remaining(), end - pos)
146+
csq.copyAsciiBytes(pos, pos + copyAmount, current.data, current.limit)
147+
current.limit += copyAmount
148+
pos += copyAmount
149+
if (pos >= end) return
150+
current = nextBlock()
151+
}
152+
} else {
153+
append(csq, start, end)
154+
}
155+
}
156+
157+
override fun appendUtf16(c: Char) {
158+
assert(c.code >= 0x80)
159+
if (current.remaining() < 3) {
160+
current = nextBlock()
161+
}
162+
if (c.code < 0x800) {
163+
current.data[current.limit++] = (0xff and (0xC0 or (c.code shr 6))).toByte()
164+
current.data[current.limit++] = (0xff and (0x80 or (c.code and 0x3F))).toByte()
165+
} else if (c.code < 0x10000) {
166+
current.data[current.limit++] = (0xff and (0xE0 or (c.code shr 12))).toByte()
167+
current.data[current.limit++] = (0xff and (0x80 or (c.code shr 6 and 0x3F))).toByte()
168+
current.data[current.limit++] = (0xff and (0x80 or (c.code and 0x3F))).toByte()
169+
}
170+
}
171+
172+
override fun appendUtf16Surrogate(leadSurrogate: Char, trailSurrogate: Char) {
173+
// Here we must convert a UTF-16 surrogate pair to UTF-8 bytes.
174+
val c = _Private_IonConstants.makeUnicodeScalar(leadSurrogate.code, trailSurrogate.code)
175+
assert(c >= 0x10000)
176+
if (current.remaining() < 4) {
177+
current = nextBlock()
178+
}
179+
current.data[current.limit++] = (0xff and (0xF0 or (c shr 18))).toByte()
180+
current.data[current.limit++] = (0xff and (0x80 or (c shr 12 and 0x3F))).toByte()
181+
current.data[current.limit++] = (0xff and (0x80 or (c shr 6 and 0x3F))).toByte()
182+
current.data[current.limit++] = (0xff and (0x80 or (c and 0x3F))).toByte()
183+
}
184+
185+
/** Helper function to wrap [java.lang.String.getBytes]. */
186+
private fun String.copyAsciiBytes(srcBegin: Int, srcEnd: Int, dst: ByteArray, dstBegin: Int) {
187+
// Using deprecated String.getBytes intentionally, since it is
188+
// correct behavior in this case, and much faster.
189+
(this as java.lang.String).getBytes(srcBegin, srcEnd, dst, dstBegin)
190+
}
191+
}

0 commit comments

Comments
 (0)