Adds stubs for BytecodeGenerators and BytecodeIonReader and core pieces for their interactions

build.gradle.kts

@@ -82,6 +82,9 @@ dependencies {
     testImplementation("pl.pragmatists:JUnitParams:1.1.1")
     testImplementation("com.google.code.tempus-fugit:tempus-fugit:1.1")
     testImplementation("com.github.luben:zstd-jni:1.5.6-5")
+
+    // For @SuppressFBWarning annotation. Only needed for SpotBugs static analysis.
+    compileOnly("com.github.spotbugs:spotbugs-annotations:4.9.5")
 }
 
 group = "com.amazon.ion"

config/proguard/rules.pro

@@ -3,3 +3,5 @@
 -dontoptimize
 -dontobfuscate
 -dontwarn java.sql.Timestamp
+# We don't need this at runtime, so it's okay if it's missing.
+-dontwarn edu.umd.cs.findbugs.annotations.SuppressFBWarnings

src/main/java/com/amazon/ion/bytecode/BytecodeGenerator.kt

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode
+
+import com.amazon.ion.Decimal
+import com.amazon.ion.Timestamp
+import com.amazon.ion.bytecode.util.AppendableConstantPoolView
+import com.amazon.ion.bytecode.util.BytecodeBuffer
+import java.math.BigInteger
+
+/**
+ * Abstracts a particular input source (e.g. [ByteArray], [ByteBuffer][java.nio.ByteBuffer]) and Ion version out
+ * of the [BytecodeIonReader].
+ *
+ * The BytecodeGenerator serves as an adapter layer between the [BytecodeIonReader] and various Ion data sources,
+ * enabling efficient reading of Ion data by converting it to bytecode instructions. This abstraction allows the
+ * reader to work with different input formats and Ion versions while minimizing branching in the hot paths. It
+ * also allows the source data to be read using a "push" model (in chunks), decoupling it from the "pull" model
+ * of the IonReader API.
+ *
+ * ## Usage Pattern
+ *
+ * The typical interaction flow is:
+ * 1. [BytecodeIonReader] calls [refill] to populate its bytecode buffer with instructions
+ * 2. As the reader processes bytecode, it calls the appropriate `read*Reference` methods to resolve
+ *    scalar values that reference the original source data
+ * 3. When an IVM is encountered, [getGeneratorForMinorVersion] may be called to switch versions
+ *
+ * ## Implementation Notes
+ *
+ * In the future, it might be possible to push all the encoding context management into this layer, which _might_
+ * provide some performance benefits by reducing the need to switch context between the BytecodeGenerator and the BytecodeIonReader.
+ *
+ * It would be possible to simplify this interface by replacing all the `read*` functions with a single function, such as this:
+ * ```
+ * fun <T> readReference(instruction: Int, operand: Int): T
+ * ```
+ * However, this seems like it would probably have a negative impact on the throughput of the reader because the call-site
+ * for the function is already in a branch for that specific kind of reference, so the single function approach would
+ * require additional branching inside the implementation of `readReference`.
+ *
+ * If we add references with int64 positions, add overrides of the `read*` methods that support a `long` position.
+ */
+internal interface BytecodeGenerator {
+
+    // TODO: Does this method need to return the symbol table and/or constant pool as well?
+    //       No, we're not going to update the encoding context in the bytecode generator.
+    //       That might limit the applicability because the bytecode needs to contain directives
+    //       for all possible Ion Versions... but we'll deal with that later.
+    /**
+     * Refills [destination] with one or more top-level user values and optionally
+     * one system value. When a system value (symbol table, directive, IVM) is encountered,
+     * no more top-level values may be filled into the destination.
+     *
+     * If there is incomplete data, and no values can be returned, it should
+     * throw IncompleteDataException (if streaming source) or IonException (if
+     * fully-buffered source).
+     */
+    fun refill(
+        /** The BytecodeBuffer that is to be filled with the bytecode */
+        destination: BytecodeBuffer,
+        /** A container for holding instances of eagerly materialized values, such as those in template definitions. */
+        constantPool: AppendableConstantPoolView,
+        /** Bytecode for each macro in the effective macro table */
+        macroSrc: IntArray,
+        /**
+         * A lookup table indicating for each macro address, where to find the first
+         * instruction for that macro in [macroSrc]. For example, to read the bytecode for the macro,
+         * you would do something like this:
+         * ```
+         * var i = macroIndices[macroAddress]
+         * var currentInstruction = macroSrc[i++]
+         * while (currentInstruction.
+         * ```
+         */
+        macroIndices: IntArray,
+        /** The current symbol table */
+        symTab: Array<String?>,
+    )
+
+    fun readBigIntegerReference(position: Int, length: Int): BigInteger
+    fun readDecimalReference(position: Int, length: Int): Decimal
+    fun readShortTimestampReference(position: Int, opcode: Int): Timestamp
+    fun readTimestampReference(position: Int, length: Int): Timestamp
+    fun readTextReference(position: Int, length: Int): String
+    fun readBytesReference(position: Int, length: Int): ByteArray
+
+    /** The Ion Minor Version supported by this [BytecodeGenerator] */
+    fun ionMinorVersion(): Int
+
+    /**
+     * When the [BytecodeIonReader] encounters an IVM that requires a version change, it will call this method.
+     * Implementations must return a BytecodeGenerator that supports the requested version and is positioned
+     * to continue compiling bytecode exactly where this BytecodeGenerator left off.
+     */
+    fun getGeneratorForMinorVersion(minorVersion: Int): BytecodeGenerator = throw UnsupportedOperationException()
+}

src/main/java/com/amazon/ion/bytecode/BytecodeIonReader.kt

@@ -0,0 +1,14 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+
+/**
+ * TODO: This class should implement [IonReader] for the Bytecode IR.
+ */
+internal class BytecodeIonReader
+@SuppressFBWarnings("URF_UNREAD_FIELD", justification = "field will be read once this class is implemented")
+constructor(
+    private var bytecodeGenerator: BytecodeGenerator,
+)

src/main/java/com/amazon/ion/bytecode/bin10/ByteArrayBytecodeGenerator10.kt

@@ -0,0 +1,15 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.bin10
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+
+@SuppressFBWarnings("EI_EXPOSE_REP2", justification = "constructor does not make a defensive copy of source as a performance optimization")
+internal class ByteArrayBytecodeGenerator10
+@SuppressFBWarnings("URF_UNREAD_FIELD", justification = "field will be read once this class is implemented")
+constructor(
+    private val source: ByteArray,
+    private var i: Int,
+) {
+    // TODO: This should implement BytecodeGenerator
+}

src/main/java/com/amazon/ion/bytecode/bin10/InputStreamBytecodeGenerator10.kt

@@ -0,0 +1,15 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.bin10
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+import java.io.InputStream
+
+internal class InputStreamBytecodeGenerator10
+@SuppressFBWarnings("URF_UNREAD_FIELD", justification = "field will be read once this class is implemented")
+constructor(
+    private val source: InputStream,
+    private var i: Long,
+) {
+    // TODO: This should implement BytecodeGenerator
+}

src/main/java/com/amazon/ion/bytecode/bin11/ByteArrayBytecodeGenerator11.kt

@@ -0,0 +1,15 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.bin11
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+
+@SuppressFBWarnings("EI_EXPOSE_REP2", justification = "constructor does not make a defensive copy of source as a performance optimization")
+internal class ByteArrayBytecodeGenerator11
+@SuppressFBWarnings("URF_UNREAD_FIELD", justification = "field will be read once this class is implemented")
+constructor(
+    private val source: ByteArray,
+    private var i: Int,
+) {
+    // TODO: This should implement BytecodeGenerator
+}

src/main/java/com/amazon/ion/bytecode/bin11/InputStreamBytecodeGenerator11.kt

@@ -0,0 +1,15 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.bin11
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+import java.io.InputStream
+
+internal class InputStreamBytecodeGenerator11
+@SuppressFBWarnings("URF_UNREAD_FIELD", justification = "field will be read once this class is implemented")
+constructor(
+    private val source: InputStream,
+    private var i: Long,
+) {
+    // TODO: This should implement BytecodeGenerator
+}

src/main/java/com/amazon/ion/bytecode/bin11/OpCode.kt

@@ -0,0 +1,142 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.bin11
+
+/**
+ * Constants representing Ion 1.1 opcode values.
+ *
+ * Not all opcodes are represented here—when there are a range of opcodes for a particular thing (macro addresses or
+ * length prefixed values), we don't necessarily want to write code for each of these opcodes individually.
+ *
+ * For dealing with tagless encodings, there are a few supplemental/replacement opcodes that have a `TE` prefix (for
+ * tagless encoding) on the name of the constant. See also [TaglessScalarType].
+ *
+ * TODO: Consider whether there is some other location more suitable for this class.
+ */
+internal object OpCode {
+    const val MACRO_0 = 0x00
+
+    const val MACRO_47 = 0x47
+
+    const val EXTENSIBLE_MACRO_ADDRESS_0 = 0x48
+    const val EXTENSIBLE_MACRO_ADDRESS_7 = 0x4F
+
+    const val SYMBOL_SID_FLEX_0 = 0x50
+    const val SYMBOL_SID_FLEX_7 = 0x57
+
+    const val ANNOTATION_SID = 0x58
+    const val ANNOTATION_TEXT = 0x59
+
+    const val RESERVED_5A = 0x5A
+
+    const val TAGLESS_ELEMENT_LIST = 0x5B
+    const val TAGLESS_ELEMENT_SEXP = 0x5C
+
+    const val RESERVED_5D = 0x5D
+    const val RESERVED_5E = 0x5E
+    const val RESERVED_5F = 0x5F
+
+    const val INT_0 = 0x60
+    const val INT_8 = 0x61
+    const val INT_16 = 0x62
+    const val INT_24 = 0x63
+    const val INT_32 = 0x64
+    const val INT_40 = 0x65
+    const val INT_48 = 0x66
+    const val INT_56 = 0x67
+    const val INT_64 = 0x68
+
+    const val RESERVED_69 = 0x69
+
+    const val FLOAT_0 = 0x6A
+    const val FLOAT_16 = 0x6B
+    const val FLOAT_32 = 0x6C
+    const val FLOAT_64 = 0x6D
+
+    const val BOOL_TRUE = 0x6E
+    const val BOOL_FALSE = 0x6F
+
+    const val DECIMAL_0 = 0x70
+    const val DECIMAL_LENGTH_15 = 0x7F
+
+    const val TIMESTAMP_YEAR_PRECISION = 0x80
+    const val TIMESTAMP_MONTH_PRECISION = 0x81
+    const val TIMESTAMP_DAY_PRECISION = 0x82
+    const val TIMESTAMP_MINUTE_PRECISION = 0x83
+    const val TIMESTAMP_SECOND_PRECISION = 0x84
+    const val TIMESTAMP_MILLIS_PRECISION = 0x85
+    const val TIMESTAMP_MICROS_PRECISION = 0x86
+    const val TIMESTAMP_NANOS_PRECISION = 0x87
+    const val TIMESTAMP_MINUTE_PRECISION_WITH_OFFSET = 0x88
+    const val TIMESTAMP_SECOND_PRECISION_WITH_OFFSET = 0x89
+    const val TIMESTAMP_MILLIS_PRECISION_WITH_OFFSET = 0x8A
+    const val TIMESTAMP_MICROS_PRECISION_WITH_OFFSET = 0x8B
+    const val TIMESTAMP_NANOS_PRECISION_WITH_OFFSET = 0x8C
+
+    const val RESERVED_8D = 0x8D
+
+    const val NULL_NULL = 0x8E
+    const val TYPED_NULL = 0x8F
+
+    const val STRING_LENGTH_0 = 0x90
+    const val STRING_LENGTH_15 = 0x9F
+
+    const val SYMBOL_LENGTH_0 = 0xA0
+    const val SYMBOL_LENGTH_15 = 0xAF
+
+    const val LIST_LENGTH_0 = 0xB0
+    const val LIST_LENGTH_15 = 0xBF
+
+    const val SEXP_LENGTH_0 = 0xC0
+    const val SEXP_LENGTH_15 = 0xCF
+
+    const val STRUCT_LENGTH_0 = 0xD0
+    const val RESERVED_D1 = 0xD1
+    const val STRUCT_LENGTH_15 = 0xDF
+
+    const val IVM = 0xE0
+    const val DIRECTIVE_SET_SYMBOLS = 0xE1
+    const val DIRECTIVE_ADD_SYMBOLS = 0xE2
+    const val DIRECTIVE_SET_MACROS = 0xE3
+    const val DIRECTIVE_ADD_MACROS = 0xE4
+    const val DIRECTIVE_USE = 0xE5
+    const val DIRECTIVE_MODULE = 0xE6
+    const val DIRECTIVE_ENCODING = 0xE7
+    const val TAGGED_PLACEHOLDER = 0xE8
+    const val TAGGED_PLACEHOLDER_WITH_DEFAULT = 0xE9
+    const val TAGLESS_PLACEHOLDER = 0xEA
+
+    const val NO_ARGUMENT = 0xEB
+    const val NOP = 0xEC
+    const val NOP_L = 0xED
+
+    const val STRUCT_SWITCH_MODES = 0xEE
+    const val DELIMITED_CONTAINER_END = 0xEF
+
+    const val DELIMITED_LIST = 0xF0
+    const val DELIMITED_SEXP = 0xF1
+    const val DELIMITED_STRUCT_SID_MODE = 0xF2
+    const val DELIMITED_STRUCT_FS_MODE = 0xF3
+
+    const val LENGTH_PREFIXED_MACRO_INVOCATION = 0xF4
+    const val VARIABLE_LENGTH_INTEGER = 0xF5
+    const val VARIABLE_LENGTH_DECIMAL = 0xF6
+    const val VARIABLE_LENGTH_TIMESTAMP = 0xF7
+    const val VARIABLE_LENGTH_STRING = 0xF8
+    const val VARIABLE_LENGTH_SYMBOL = 0xF9
+    const val VARIABLE_LENGTH_LIST = 0xFA
+    const val VARIABLE_LENGTH_SEXP = 0xFB
+    const val VARIABLE_LENGTH_STRUCT_SID_MODE = 0xFC
+    const val VARIABLE_LENGTH_STRUCT_FS_MODE = 0xFD
+    const val VARIABLE_LENGTH_BLOB = 0xFE
+    const val VARIABLE_LENGTH_CLOB = 0xFF
+
+    const val TE_FLEX_INT = 0x60
+    const val TE_FLEX_UINT = 0xE0
+    const val TE_UINT_8 = 0xE1
+    const val TE_UINT_16 = 0xE2
+    const val TE_UINT_32 = 0xE4
+    const val TE_UINT_64 = 0xE8
+    const val TE_SYMBOL_SID = 0xEA
+    const val TE_SYMBOL_FS = 0xEE
+}

src/main/java/com/amazon/ion/bytecode/ir/Debugger.kt

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.bytecode.ir
 
-import com.amazon.ion._private_.SuppressFBWarnings
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
 import java.util.function.Consumer
 
 /**

src/main/java/com/amazon/ion/bytecode/util/AppendableConstantPoolView.kt

@@ -0,0 +1,13 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.util
+
+/**
+ * A view of a [ConstantPool] that allows read and append operations.
+ */
+interface AppendableConstantPoolView {
+    /** Adds a value to the constant pool, returning the index assigned to the value. */
+    fun add(value: Any?): Int
+    /** Retrieves a value from the constant pool. */
+    fun get(i: Int): Any?
+}

src/main/java/com/amazon/ion/bytecode/util/ByteSlice.kt

@@ -0,0 +1,30 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.bytecode.util
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
+
+/**
+ * Light-weight representation of a portion of a [ByteArray].
+ *
+ * Positions are relative to `bytes`, not the underlying data stream.
+ *
+ * This is not intended to be exposed publicly, but it might end up needing to be exposed in order to expose a
+ * template-building API. If it does get exposed, we need to revisit the location of this class.
+ */
+@SuppressFBWarnings(
+    value = ["EI_EXPOSE_REP"],
+    justification = "This class is only for internal use. Exposing the internal array is intentional to avoid the performance overhead of copying it."
+)
+internal class ByteSlice(
+    val bytes: ByteArray,
+    val startInclusive: Int,
+    val endExclusive: Int
+) {
+    val length = endExclusive - startInclusive
+
+    /**
+     * Convenience method to create a new array that is a copy of this data represented by this [ByteSlice].
+     */
+    fun newByteArray() = bytes.copyOfRange(startInclusive, endExclusive)
+}