Skip to content

Commit 405f760

Browse files
popemattpopematt
authored
Adds some utilities for Ion 1.0 BytecodeGenerators (#1124)
1 parent 0652035 commit 405f760

7 files changed

Lines changed: 687 additions & 0 deletions

File tree

‎src/main/java/com/amazon/ion/bytecode/bin10/TypeIdHelper.kt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package com.amazon.ion.bytecode.bin10
4+
5+
import com.amazon.ion.IonType
6+
import com.amazon.ion.bytecode.ir.OperationKind
7+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
8+
9+
internal object TypeIdHelper {
10+
11+
/**
12+
* Returns the IonType for a legal Ion 1.0 typeId.
13+
*
14+
* The returned value is undefined when attempting to use this function for illegal type IDs.
15+
*/
16+
@JvmStatic
17+
fun ionTypeForTypeId(typeId: Int): IonType? {
18+
return when (typeId shr 4) {
19+
0x0 -> if (typeId == 0x0F) IonType.NULL else null
20+
0x1 -> IonType.BOOL
21+
0x2, 0x3 -> IonType.INT
22+
0x4 -> IonType.FLOAT
23+
0x5 -> IonType.DECIMAL
24+
0x6 -> IonType.TIMESTAMP
25+
0x7 -> IonType.SYMBOL
26+
0x8 -> IonType.STRING
27+
0x9 -> IonType.CLOB
28+
0xA -> IonType.BLOB
29+
0xB -> IonType.LIST
30+
0xC -> IonType.SEXP
31+
0xD -> IonType.STRUCT
32+
else -> null
33+
}
34+
}
35+
36+
/**
37+
* Returns the [OperationKind] for an Ion 1.0 typeId.
38+
*/
39+
@JvmStatic
40+
fun operationKindForTypeId(typeId: Int): Int = typeToOperationKindLookup[typeId]
41+
42+
private val typeToOperationKindLookup = IntArray(256) { initOperationKindForType(it) }
43+
44+
private fun initOperationKindForType(state: Int): Int {
45+
return when (state) {
46+
in 0x00..0x0E -> OperationKind.UNSET
47+
0x0F -> OperationKind.NULL
48+
0x10, 0x11, 0x1F -> OperationKind.BOOL
49+
in 0x20..0x2F -> OperationKind.INT
50+
in 0x31..0x3F -> OperationKind.INT
51+
0x40, 0x44, 0x48, 0x4F -> OperationKind.FLOAT
52+
in 0x50..0x5F -> OperationKind.DECIMAL
53+
in 0x62..0x6F -> OperationKind.TIMESTAMP
54+
in 0x70..0x7F -> OperationKind.SYMBOL
55+
in 0x80..0x8F -> OperationKind.STRING
56+
in 0x90..0x9F -> OperationKind.CLOB
57+
in 0xA0..0xAF -> OperationKind.BLOB
58+
in 0xB0..0xBF -> OperationKind.LIST
59+
in 0xC0..0xCF -> OperationKind.SEXP
60+
0xD0, in 0xD2..0xDF -> OperationKind.STRUCT
61+
0xE0 -> OperationKind.IVM
62+
in 0xE3..0xEE -> OperationKind.ANNOTATIONS
63+
// Everything else: 12..1E, 30, D1, EF, F0..FF, illegal timestamp, float, and annotations sizes
64+
else -> OperationKind.UNSET
65+
}
66+
}
67+
68+
/**
69+
* A lookup table containing the value length for Ion 1.0 type IDs.
70+
* A value of -1 indicates that the length follows as a `VarUInt`.
71+
* A value of -2 indicates that the typeId is not a valid Ion 1.0 typeId.
72+
*
73+
* TODO(perf): If there's not a noticeable amount of overhead, we should hide this behind a method instead of
74+
* exposing it directly and suppressing the spotbugs error.
75+
*/
76+
@JvmStatic
77+
@get:SuppressFBWarnings("MS_EXPOSE_REP", justification = "it is exposed for internal use only as a performance optimization")
78+
val TYPE_LENGTHS = IntArray(256) { initTypeLength(it) }
79+
80+
@JvmStatic
81+
@OptIn(ExperimentalStdlibApi::class)
82+
private fun initTypeLength(typeId: Int): Int {
83+
return when (typeId) {
84+
// The length of all of these is determined entirely by the low-nibble of the typeId.
85+
0x00, 0x10, 0x20, 0x30, 0x40, 0x50, /* */ 0x70, 0x80, 0x90, 0xA0, 0xB0, 0xC0, 0xD0 /* */ -> 0
86+
0x01, /* */ 0x21, 0x31, /* */ 0x51, /* */ 0x71, 0x81, 0x91, 0xA1, 0xB1, 0xC1 /* */ /* */ -> 1
87+
0x02, 0x12, 0x22, 0x32, /* */ 0x52, 0x62, 0x72, 0x82, 0x92, 0xA2, 0xB2, 0xC2, 0xD2 /* */ -> 2
88+
0x03, 0x13, 0x23, 0x33, /* */ 0x53, 0x63, 0x73, 0x83, 0x93, 0xA3, 0xB3, 0xC3, 0xD3, 0xE3 -> 3
89+
0x04, 0x14, 0x24, 0x34, 0x44, 0x54, 0x64, 0x74, 0x84, 0x94, 0xA4, 0xB4, 0xC4, 0xD4, 0xE4 -> 4
90+
0x05, 0x15, 0x25, 0x35, /* */ 0x55, 0x65, 0x75, 0x85, 0x95, 0xA5, 0xB5, 0xC5, 0xD5, 0xE5 -> 5
91+
0x06, 0x16, 0x26, 0x36, /* */ 0x56, 0x66, 0x76, 0x86, 0x96, 0xA6, 0xB6, 0xC6, 0xD6, 0xE6 -> 6
92+
0x07, 0x17, 0x27, 0x37, /* */ 0x57, 0x67, 0x77, 0x87, 0x97, 0xA7, 0xB7, 0xC7, 0xD7, 0xE7 -> 7
93+
0x08, 0x18, 0x28, 0x38, 0x48, 0x58, 0x68, 0x78, 0x88, 0x98, 0xA8, 0xB8, 0xC8, 0xD8, 0xE8 -> 8
94+
0x09, 0x19, 0x29, 0x39, /* */ 0x59, 0x69, 0x79, 0x89, 0x99, 0xA9, 0xB9, 0xC9, 0xD9, 0xE9 -> 9
95+
0x0A, 0x1A, 0x2A, 0x3A, /* */ 0x5A, 0x6A, 0x7A, 0x8A, 0x9A, 0xAA, 0xBA, 0xCA, 0xDA, 0xEA -> 10
96+
0x0B, 0x1B, 0x2B, 0x3B, /* */ 0x5B, 0x6B, 0x7B, 0x8B, 0x9B, 0xAB, 0xBB, 0xCB, 0xDB, 0xEB -> 11
97+
0x0C, 0x1C, 0x2C, 0x3C, /* */ 0x5C, 0x6C, 0x7C, 0x8C, 0x9C, 0xAC, 0xBC, 0xCC, 0xDC, 0xEC -> 12
98+
0x0D, 0x1D, 0x2D, 0x3D, /* */ 0x5D, 0x6D, 0x7D, 0x8D, 0x9D, 0xAD, 0xBD, 0xCD, 0xDD, 0xED -> 13
99+
0x0E, 0x1E, 0x2E, 0x3E, /* */ 0x5E, 0x6E, 0x7E, 0x8E, 0x9E, 0xAE, 0xBE, 0xCE, 0xDE, 0xEE -> -1
100+
// Bool True
101+
0x11 -> 0
102+
// Nulls
103+
0x0F, 0x1F, 0x2F, 0x3F, 0x4F, 0x5F, 0x6F, 0x7F, 0x8F, 0x9F, 0xAF, 0xBF, 0xCF, 0xDF -> 0
104+
// IVM
105+
0xE0 -> 3 // ...3 more than the typeId byte.
106+
107+
// Reserved and/or illegal typeIds
108+
0xEF, 0x60, 0x61, 0xD1, 0xE1, 0xE2,
109+
in 0x41..0x4E, // Illegal float lengths. 0x44 and 0x48 are trapped in the earlier condition.
110+
in 0xF0..0xFF -> -2
111+
else -> TODO("This should be unreachable: ${typeId.toHexString()}")
112+
}
113+
}
114+
}

‎src/main/java/com/amazon/ion/bytecode/bin10/ValueHelpers.kt‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package com.amazon.ion.bytecode.bin10
4+
5+
import com.amazon.ion.Decimal
6+
import com.amazon.ion.IonException
7+
import com.amazon.ion.Timestamp
8+
import java.math.BigDecimal
9+
import java.math.BigInteger
10+
import kotlin.experimental.and
11+
12+
/**
13+
* Given a typeId in the range 0x20..0x3F, returns either -1 or 1.
14+
* This uses some clever bit twiddling to avoid any branching.
15+
*
16+
* (Yes, it's obtuse looking, but it works.)
17+
*
18+
* TODO(perf): Once reader is implemented enough for benchmarks, benchmark this vs using if/else.
19+
*/
20+
internal fun signForIntTypeId(typeId: Int): Int = (((typeId shr 4) shl 31) shr 31) or 1
21+
22+
/**
23+
* Return either -1 or 1 based on the sign bit of the given byte. This uses some bit manipulation to avoid any branching.
24+
*/
25+
internal fun getSignumValueFromLeadingSignBit(byte: Byte): Int = byte.toInt().shr(7).shl(1) + 1
26+
27+
/**
28+
* Reads a timestamp value from the given byte array.
29+
*/
30+
internal fun readTimestampReference(valueBytes: ByteArray, position: Int, length: Int): Timestamp {
31+
var p = position
32+
val end = position + length
33+
34+
val offset: Int? = if (valueBytes[p].toInt() and 0xFF == 0xC0) {
35+
p++
36+
null
37+
} else {
38+
val offsetValueAndLength = VarIntHelper.readVarIntValueAndLength(valueBytes, p)
39+
p += offsetValueAndLength.toInt() and 0xFF
40+
(offsetValueAndLength shr 8).toInt()
41+
}
42+
val yearValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
43+
p += yearValueAndLength.toInt() and 0xFF
44+
val year = (yearValueAndLength shr 8).toInt()
45+
var month = 0
46+
var day = 0
47+
var hour = 0
48+
var minute = 0
49+
var second = 0
50+
var fractionalSecond: BigDecimal? = null
51+
var precision = Timestamp.Precision.YEAR
52+
if (p < end) {
53+
val monthValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
54+
p += monthValueAndLength.toInt() and 0xFF
55+
month = (monthValueAndLength shr 8).toInt()
56+
precision = Timestamp.Precision.MONTH
57+
if (p < end) {
58+
val dayValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
59+
p += dayValueAndLength.toInt() and 0xFF
60+
day = (dayValueAndLength shr 8).toInt()
61+
precision = Timestamp.Precision.DAY
62+
if (p < end) {
63+
val hourValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
64+
p += hourValueAndLength.toInt() and 0xFF
65+
hour = (hourValueAndLength shr 8).toInt()
66+
if (p >= end) {
67+
throw IonException("Timestamps may not specify hour without specifying minute.")
68+
}
69+
70+
val minuteValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
71+
p += minuteValueAndLength.toInt() and 0xFF
72+
minute = (minuteValueAndLength shr 8).toInt()
73+
precision = Timestamp.Precision.MINUTE
74+
if (p < end) {
75+
val secondValueAndLength = VarIntHelper.readVarUIntValueAndLength(valueBytes, p)
76+
p += secondValueAndLength.toInt() and 0xFF
77+
second = (secondValueAndLength shr 8).toInt()
78+
precision = Timestamp.Precision.SECOND
79+
if (p < end) {
80+
fractionalSecond = readDecimalReference(valueBytes, p, end)
81+
if (fractionalSecond.scale() < 0) {
82+
fractionalSecond = fractionalSecond.setScale(0)
83+
}
84+
}
85+
}
86+
}
87+
}
88+
}
89+
try {
90+
return Timestamp.createFromUtcFields(
91+
precision,
92+
year,
93+
month,
94+
day,
95+
hour,
96+
minute,
97+
second,
98+
fractionalSecond,
99+
offset
100+
)
101+
} catch (e: IllegalArgumentException) {
102+
throw IonException("Illegal timestamp encoding at $position.", e)
103+
}
104+
}
105+
106+
/**
107+
* Reads a Decimal value from the given byte array.
108+
*/
109+
internal fun readDecimalReference(valueBytes: ByteArray, position: Int, end: Int): Decimal {
110+
var p = position
111+
val exponentValueAndLength = VarIntHelper.readVarIntValueAndLength(valueBytes, p)
112+
p += exponentValueAndLength.toInt() and 0xFF
113+
val scale = -(exponentValueAndLength shr 8).toInt()
114+
115+
val coefficientLength = end - p
116+
return if (coefficientLength > 0) {
117+
// TODO: See if we can have a shared set of reusable buffers for this instead of allocating a copy.
118+
val bytes = valueBytes.copyOfRange(p, p + coefficientLength)
119+
120+
// Get the signum
121+
val signum = getSignumValueFromLeadingSignBit(bytes[0])
122+
// Clear the sign bit
123+
bytes[0] = bytes[0] and 0x7F
124+
// Construct the BigInteger
125+
val coefficient = BigInteger(signum, bytes)
126+
if (coefficient == BigInteger.ZERO && signum == -1) {
127+
Decimal.negativeZero(scale)
128+
} else {
129+
Decimal.valueOf(BigInteger(signum, bytes), scale)
130+
}
131+
} else {
132+
Decimal.valueOf(BigInteger.ZERO, scale)
133+
}
134+
}

‎src/main/java/com/amazon/ion/bytecode/bin10/VarIntHelper.kt‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package com.amazon.ion.bytecode.bin10
4+
5+
import com.amazon.ion.IonException
6+
7+
object VarIntHelper {
8+
9+
private const val TERMINATION_BIT_MASK = 0b10000000
10+
private const val MASK_7_BITS = 0b01111111
11+
12+
/**
13+
* Returns an unsigned integer up to 7 bytes, with an 1 byte integer signifying how many varuint bytes were used in its encoding.
14+
*/
15+
@JvmStatic
16+
fun readVarUIntValueAndLength(source: ByteArray, position: Int): Long {
17+
val currentByte: Int = source[position].toInt()
18+
val result = (currentByte and MASK_7_BITS).toLong()
19+
return if (currentByte and TERMINATION_BIT_MASK != 0) {
20+
(result shl 8) or 1L
21+
} else {
22+
readVarUIntValueAndLength2(source, position + 1, result)
23+
}
24+
}
25+
26+
@JvmStatic
27+
private fun readVarUIntValueAndLength2(source: ByteArray, position: Int, partialResult: Long): Long {
28+
val currentByte: Int = source.get(position).toInt()
29+
val result = (partialResult shl 7) or (currentByte and MASK_7_BITS).toLong()
30+
if (currentByte and TERMINATION_BIT_MASK != 0) {
31+
return (result shl 8) or 2
32+
} else {
33+
return readVarUIntValueAndLength3Plus(source, position + 1, result)
34+
}
35+
}
36+
37+
@JvmStatic
38+
private fun readVarUIntValueAndLength3Plus(source: ByteArray, position: Int, partialResult: Long): Long {
39+
var currentByte: Int
40+
var result = partialResult
41+
var p = position
42+
var length = 2
43+
do {
44+
length++
45+
if (length > 7) throw IonException("VarUInt value is too large")
46+
currentByte = source.get(p++).toInt()
47+
result = (result shl 7) or (currentByte and MASK_7_BITS).toLong()
48+
} while (currentByte and TERMINATION_BIT_MASK == 0)
49+
50+
return (result shl 8) or length.toLong()
51+
}
52+
53+
/**
54+
* Returns a signed integer up to 7 bytes, with an 1 byte integer signifying how many varuint bytes were used in its encoding.
55+
*/
56+
@JvmStatic
57+
fun readVarIntValueAndLength(source: ByteArray, position: Int): Long {
58+
var p = position
59+
60+
var length = 1
61+
try {
62+
var currentByte = source[p++].toInt() and 0xFF
63+
var result = (currentByte and 0b00111111).toLong()
64+
val sign = getSignumValueFromVarIntSignBit(currentByte)
65+
while (currentByte and TERMINATION_BIT_MASK == 0) {
66+
length++
67+
if (length > 7) throw IonException("VarInt value is too large")
68+
currentByte = source[p++].toInt()
69+
result = (result shl 7) or (currentByte and MASK_7_BITS).toLong()
70+
}
71+
return ((sign * result) shl 8) or length.toLong()
72+
} catch (e: ArrayIndexOutOfBoundsException) {
73+
throw IonException("Incomplete VarInt at position $position", e)
74+
}
75+
}
76+
77+
/**
78+
* Return either -1 or 1 based on the sign bit of the given byte. This uses some bit manipulation to avoid any branching.
79+
*
80+
* Visible only for testing.
81+
*/
82+
@JvmStatic
83+
internal fun getSignumValueFromVarIntSignBit(byte: Int): Int = byte.shl(25).shr(31).shl(1) + 1
84+
}

‎src/test/java/com/amazon/ion/TextToBinaryUtils.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ object TextToBinaryUtils {
2424
return bytesAsBytes
2525
}
2626

27+
/**
28+
* Converts a string of binary octets, such as "10010111 00010011", to a byte array.
29+
*/
30+
@JvmStatic
31+
fun String.binaryStringToByteArray(): ByteArray {
32+
return octetStringToByteArray(this, 2)
33+
}
34+
2735
/**
2836
* Converts a string of hex octets, such as "BE EF", to a byte array.
2937
*/

0 commit comments

Comments
 (0)