From 18495cd1bc1c7c4f9f7e1ae6bcd39c74254adbb8 Mon Sep 17 00:00:00 2001 From: Matan Ziv-Av Date: Sat, 3 Sep 2022 13:58:33 +0300 Subject: [PATCH 01/15] Fixed: Clear LineWrap flag when clearing lines using blockSet --- .../src/main/java/com/termux/terminal/TerminalBuffer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java index 21d6518785..796afa0ba4 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java @@ -439,9 +439,13 @@ public void blockSet(int sx, int sy, int w, int h, int val, long style) { throw new IllegalArgumentException( "Illegal arguments! blockSet(" + sx + ", " + sy + ", " + w + ", " + h + ", " + val + ", " + mColumns + ", " + mScreenRows + ")"); } - for (int y = 0; y < h; y++) + for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) setChar(sx + x, sy + y, val, style); + if (sx + w == mColumns && val == ' ') { + clearLineWrap(sy + y); + } + } } public TerminalRow allocateFullLineIfNecessary(int row) { From 439b0b693f7c2ce90677326c31724763f1ce3e13 Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Tue, 31 Mar 2026 00:40:38 +0500 Subject: [PATCH 02/15] Fixed: Revert using `Canvas.drawText()` instead of `Canvas.drawTextRun` only on Android 5 as later function is not available on it and will cause a runtime crash The change was done in 6c00f1fc61 via #2997 --- .../src/main/java/com/termux/view/TerminalRenderer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java index a4bef7d37c..5d29687fb3 100644 --- a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java +++ b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java @@ -4,6 +4,7 @@ import android.graphics.Paint; import android.graphics.PorterDuff; import android.graphics.Typeface; +import android.os.Build; import com.termux.terminal.TerminalBuffer; import com.termux.terminal.TerminalEmulator; @@ -233,7 +234,11 @@ private void drawTextRun(Canvas canvas, char[] text, int[] palette, float y, int mTextPaint.setColor(foreColor); // The text alignment is the default Paint.Align.LEFT. - canvas.drawTextRun(text, startCharIndex, runWidthChars, startCharIndex, runWidthChars, left, y - mFontLineSpacingAndAscent, false, mTextPaint); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + canvas.drawTextRun(text, startCharIndex, runWidthChars, startCharIndex, runWidthChars, left, y - mFontLineSpacingAndAscent, false, mTextPaint); + } else { + canvas.drawText(text, startCharIndex, runWidthChars, left, y - mFontLineSpacingAndAscent, mTextPaint); + } } if (savedMatrix) canvas.restore(); From bdc3d72f8faa40a4acd7d2153fb8a1c1ebfdff2b Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Tue, 31 Mar 2026 00:53:26 +0500 Subject: [PATCH 03/15] Fixed: Explicitly cast cursor dimensions to float to prevent loses --- .../src/main/java/com/termux/view/TerminalRenderer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java index 5d29687fb3..214df331b3 100644 --- a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java +++ b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java @@ -209,8 +209,8 @@ private void drawTextRun(Canvas canvas, char[] text, int[] palette, float y, int if (cursor != 0) { mTextPaint.setColor(cursor); float cursorHeight = mFontLineSpacingAndAscent - mFontAscent; - if (cursorStyle == TerminalEmulator.TERMINAL_CURSOR_STYLE_UNDERLINE) cursorHeight /= 4.; - else if (cursorStyle == TerminalEmulator.TERMINAL_CURSOR_STYLE_BAR) right -= ((right - left) * 3) / 4.; + if (cursorStyle == TerminalEmulator.TERMINAL_CURSOR_STYLE_UNDERLINE) cursorHeight /= 4.f; + else if (cursorStyle == TerminalEmulator.TERMINAL_CURSOR_STYLE_BAR) right -= (((right - left) * 3) / 4.f); canvas.drawRect(left, y - cursorHeight, right, y, mTextPaint); } From 4748e230accd7f94e462c61151f2a4aa8f826bd1 Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Tue, 31 Mar 2026 22:27:47 +0500 Subject: [PATCH 04/15] Changed: Use `int` instead of `short` for space used for proper indexing --- .../src/main/java/com/termux/terminal/TerminalRow.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java index d68dc32623..8d7e6bab20 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java @@ -42,7 +42,7 @@ public final class TerminalRow { /** The text filling this terminal row. */ public char[] mText; /** The number of java chars used in {@link #mText}. */ - private short mSpaceUsed; + private int mSpaceUsed; /** If this row has been line wrapped due to text output at the end of line. */ boolean mLineWrap; /** The style bits of each cell in the row. See {@link TextStyle}. */ @@ -144,7 +144,7 @@ private boolean wideDisplayCharacterStartingAt(int column) { public void clear(long style) { Arrays.fill(mText, ' '); Arrays.fill(mStyle, style); - mSpaceUsed = (short) mColumns; + mSpaceUsed = mColumns; mHasNonOneWidthOrSurrogateChars = false; } @@ -256,7 +256,7 @@ public void setChar(int columnToSet, int codePoint, long style) { throw new IllegalArgumentException("Cannot put wide character in last column"); } else if (columnToSet == mColumns - 2) { // Truncate the line to the second part of this wide char: - mSpaceUsed = (short) newNextColumnIndex; + mSpaceUsed = newNextColumnIndex; } else { // Overwrite the contents of the next column, which mean we actually remove java characters. Due to the // check at the beginning of this method we know that we are not overwriting a wide char. From 7461709a832216b5aaecae86e273a959d6b9c58f Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Thu, 9 Apr 2026 15:35:23 +0500 Subject: [PATCH 05/15] Fixed|Changed: Explicitly pass `Base64.DEFAULT` while base64 decoding text received for setting to clipboard from terminal and fix error format typo --- .../src/main/java/com/termux/terminal/TerminalEmulator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index b0be6f3440..d4023228c3 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -2107,10 +2107,10 @@ private void doOscSetTextParameters(String bellOrStringTerminator) { case 52: // Manipulate Selection Data. Skip the optional first selection parameter(s). int startIndex = textParameter.indexOf(";") + 1; try { - String clipboardText = new String(Base64.decode(textParameter.substring(startIndex), 0), StandardCharsets.UTF_8); + String clipboardText = new String(Base64.decode(textParameter.substring(startIndex), Base64.DEFAULT), StandardCharsets.UTF_8); mSession.onCopyTextToClipboard(clipboardText); } catch (Exception e) { - Logger.logError(mClient, LOG_TAG, "OSC Manipulate selection, invalid string '" + textParameter + ""); + Logger.logError(mClient, LOG_TAG, "OSC Manipulate selection, invalid string '" + textParameter + "'"); } break; case 104: From 948ffa41c87c4074ffc81eb3da99c37b9715c83d Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Thu, 9 Apr 2026 03:19:07 +0500 Subject: [PATCH 06/15] Added|Changed|Fixed: Standardize function/variable names and add docs for OSC/DCS/APC in `TerminalEmulator`, add support for fast path for OSC/DCS terminal commands, and properly clear buffer after commands Rename `ESC_P` to `ESC_DCS`, `ESC_APC_ESCAPE` to `ESC_APC__ESC`, `doDeviceControl()` to `doDcs()`, `doOsc()` to `receiveOsc()`, `doOscEsc()` to `receiveOscEsc()`, `doApc()` to `receiveApc()`, `doApcEscape()` to `receiveApcEsc()`, `doOscSetTextParameters()` to `doOsc()`. `mOSCOrDeviceControlArgs` `StringBuilder` buffer has been renamed to `mTerminalControlArgs` as it stores args for all 3 OSC/DCS/APC commands. Additionally, for `mTerminalControlArgs`, only `setLength(0)` was called at end of commands to clear it, which will only update internal length marker and not reduce internal array capacity, and to deallocate extra memory `trimToSize()` needs to be called, which creates another smaller array. So if an OSC/APC command was ever received with `MAX_OSC_STRING_LENGTH`, then the buffer size would never be brought back down to a lower capacity (default 16bytes), which would have slightly affected performance/memory for later smaller commands. That is fixed now by `clearTerminalControlArgs()` with a newer logic for freeing memory, which is required to free memory for long commands like sixels or iterm image to be added later. `ensureTerminalControlArgsCapacity()` has also been added to allow control commands to increase buffer capacity to reduce/prevent reallocation and copying of memory as more data is received. `collectTerminalControlArgs()` has been updated with better error message, catches OOM in case caused when buffer capacity is increased when appending, and returns `false` if an error occurred like an overflow/OOM so that callers can terminate processing. This was required because `receiveApcEsc()`/`receiveOscEsc()` functions appended `ESC`/`27` and then appended `b` without checking if `collectOSCArgs()` had already failed when collecting `ESC`, triggering a second error. `collectTerminalControlArgs()` is now also used for `DCS` commands properly handle errors/exceptions. `MAX_OSC_STRING_LENGTH` has been renamed to `TERMINAL_CONTROL_ARGS__MAX_LENGTH` and the length has been increased from `8192` to `16384` to receive larger sixel data and iTerm image data. With `16384`, around 12KB images can be received with legacy `File=` command, otherwise with `8192`, any image over 6KB would have triggered an overflow. Sixel commands strings can also be split into larger parts in future with the new limit. The `TerminalEmulator.processCodePoint()` processes bytes received by the terminal, however there are around ~30 case comparisons done before `ESC_OSC` or `ESC_DCS` is processed for each byte received. While it may be necessary for general commands, some commands like sixel or iterm image data parse their own input byte, which may be random data, so a fast path has been added at start of `processCodePoint()` to call `doOsc` or `doDcs()` if enabled by the currently running command. Fast path can be enabled for OSC commands via `setOscTypeVariables()` and for `DCS` commands via `doDcs()`. OSC commands can also prevent CR/LF characters being printed by enabling `mIgnoreCrLfForOsc` via `setOscTypeVariables()`. Escape states for Terminal APC commands fast path added in f35063da to `TerminalEmulator.processCodePoint()` has also been fixed. --- .../com/termux/terminal/TerminalEmulator.java | 486 +++++++++++++++--- 1 file changed, 403 insertions(+), 83 deletions(-) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index d4023228c3..0eefd5d01e 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -12,6 +12,8 @@ * Renders text into a screen. Contains all the terminal-specific knowledge and state. Emulates a subset of the X Window * System xterm terminal, which in turn is an emulator for a subset of the Digital Equipment Corporation vt100 terminal. *

+ * See also 7-bit Code Table defined at https://vt100.net/docs/vt220-rm/chapter2.html#S2.3.1 + *

* References: *

    *
  • http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
    • @@ -41,6 +43,15 @@ public final class TerminalEmulator { /** Used for invalid data - http://en.wikipedia.org/wiki/Replacement_character#Replacement_character */ public static final int UNICODE_REPLACEMENT_CHAR = 0xFFFD; + /* + * Escape sequences starting with an ESC character. + * + * - https://vt100.net/docs/vt220-rm/chapter2.html#S2.5.1 + * - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Controls-beginning-with-ESC + * - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-C1-lparen-8-Bit-rparen-Control-Characters + * - https://en.wikipedia.org/wiki/C0_and_C1_control_codes + */ + /** Escape processing: Not currently in an escape sequence. */ private static final int ESC_NONE = 0; /** Escape processing: Have seen an ESC character - proceed to {@link #doEsc(int)} */ @@ -59,14 +70,66 @@ public final class TerminalEmulator { private static final int ESC_CSI_DOLLAR = 8; /** Escape processing: ESC % */ private static final int ESC_PERCENT = 9; - /** Escape processing: ESC ] (AKA OSC - Operating System Controls) */ + /** + * Escape processing: `ESC ]` for Operating System Command (OSC) + *

    + * `OSC` commands may be in one of the following formats: + * - `OSC Ps ; Pt BEL` where `BEL` is the bell control passed as `\a`. + * - `OSC Ps ; Pt ST` where `ST` is the string terminator passed as `ESC \`. + * `ST` is the preferred standard for modern terminals. + *

    + * If an `OSC` escape sequence is received, then {@link #mEscapeState} is set to {@link #ESC_OSC} + * and {@link #receiveOsc(int)} is called by {@link #processCodePoint(int)}. + * - By default it will add bytes received after `OSC` escape sequence to {@link #mTerminalControlArgs}. + * - If a `BEL` is received, then {@link #doOsc(String)} is called to process + * the OSC command. + * - If an `ESC` is received, then {@link #mEscapeState} is set to {@link #ESC_OSC__ESC} and + * {@link #receiveOscEsc(int)} is called for the next code point. + * - If the next code point is a `\` for `ST`, then {@link #doOsc(String)} is + * called to process the OSC command. + * - If the next code point is not a `\`, then {@link #mEscapeState} is set back to + * {@link #ESC_OSC} as `ESC` may be part of command data as so it is added to + * {@link #mTerminalControlArgs}, and for later code points {@link #receiveOsc(int)} is called + * instead. + *

    + * While an `OSC` is being received, {@link #mOscType} may be set to the command type when it + * has been fully received by {@link #setOscTypeVariables()}. + *

    + * See also {@link #mIsFastPathOsc} for enabling fast path for specific `OSC` commands if required + * via {@link #setOscTypeVariables()}. + *

    + * See also {@link #mIgnoreCrLfForOsc} to prevent printing of CR/LF characters for specific + * `OSC` commands if required via {@link #setOscTypeVariables()}. + *

    + * - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Operating-System-Commands + */ private static final int ESC_OSC = 10; - /** Escape processing: ESC ] (AKA OSC - Operating System Controls) ESC */ - private static final int ESC_OSC_ESC = 11; + /** Escape processing: `ESC` received while receiving a {@link #ESC_OSC} command. */ + private static final int ESC_OSC__ESC = 11; /** Escape processing: ESC [ > */ private static final int ESC_CSI_BIGGERTHAN = 12; - /** Escape procession: "ESC P" or Device Control String (DCS) */ - private static final int ESC_P = 13; + /** + * Escape procession: `ESC P` for Device Control String (DCS) + *

    + * `DCS` commands are in the format `DCS data ST` where `ST` is the string terminator passed as `ESC \`. + * `data` is application defined raw data without any specific standards. + *

    + * If an `DCS` escape sequence is received, then {@link #mEscapeState} is set to {@link #ESC_DCS} + * and {@link #doDcs(int)} is called by {@link #processCodePoint(int)}. + * - By default it will add bytes received after `DCS` escape sequence to {@link #mTerminalControlArgs}. + * - If an `ESC` is received by {@link #processCodePoint(int)}, then {@link #ESC_DCS__ESC} set + * to `true`. + * - If the next code point is a `\` for `ST`, then {@link #doDcs(int)} processes the DCS command. + * - If the next code point is not a `\`, then {@link #ESC_DCS__ESC} is set back to `false` as + * `ESC` may be part of command data as so it is added to {@link #mTerminalControlArgs}. + * - For certain commands like sixel commands, {@link #doDcs(int)} alters the default behaviour. + *

    + * See also {@link #mIsFastPathDcs} for enabling fast path for specific `DCS` commands if required. + *

    + *

    + * - https://vt100.net/docs/vt220-rm/chapter2.html#S2.5.3 + */ + private static final int ESC_DCS = 13; /** Escape processing: CSI > */ private static final int ESC_CSI_QUESTIONMARK_ARG_DOLLAR = 14; /** Escape processing: CSI $ARGS ' ' */ @@ -79,10 +142,31 @@ public final class TerminalEmulator { private static final int ESC_CSI_SINGLE_QUOTE = 18; /** Escape processing: CSI ! */ private static final int ESC_CSI_EXCLAMATION = 19; - /** Escape processing: "ESC _" or Application Program Command (APC). */ + /** + * Escape processing: `ESC _` for Application Program Command (APC). + *

    + * `APC` commands are in the format `APC data ST` where `ST` is the string terminator passed as `ESC \`. + * `data` is application defined raw data without any specific standards. + *

    + * If an `APC` escape sequence is received, then {@link #mEscapeState} is set to {@link #ESC_APC} + * and {@link #receiveApc(int)} is called by {@link #processCodePoint(int)}. + * - By default it will add bytes received after `APC` escape sequence to {@link #mTerminalControlArgs}. + * - If an `ESC` is received, then {@link #mEscapeState} is set to {@link #ESC_APC__ESC} and + * {@link #receiveApcEsc(int)} is called for the next code point. + * - If the next code point is a `\` for `ST`, then {@link #doApc()} is called to + * process the APC command. + * - If the next code point is not a `\`, then {@link #mEscapeState} is set back to + * {@link #ESC_APC} as `ESC` may be part of command data as so it is added to + * {@link #mTerminalControlArgs}, and for later code points {@link #receiveApc(int)} is called + * instead. + *

    + * Currently, APC commands are only parsed, but ignored as none are supported. + *

    + * - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Application-Program-Command-functions + */ private static final int ESC_APC = 20; - /** Escape processing: "ESC _" or Application Program Command (APC), followed by Escape. */ - private static final int ESC_APC_ESCAPE = 21; + /** Escape processing: `ESC` received while receiving a {@link #ESC_APC} command. */ + private static final int ESC_APC__ESC = 21; /** Escape processing: ESC [ */ private static final int ESC_CSI_UNSUPPORTED_PARAMETER_BYTE = 22; /** Escape processing: ESC [ */ @@ -91,9 +175,6 @@ public final class TerminalEmulator { /** The number of parameter arguments including colon separated sub-parameters. */ private static final int MAX_ESCAPE_PARAMETERS = 32; - /** Needs to be large enough to contain reasonable OSC 52 pastes. */ - private static final int MAX_OSC_STRING_LENGTH = 8192; - /** DECSET 1 - application cursor keys. */ private static final int DECSET_BIT_APPLICATION_CURSOR_KEYS = 1; private static final int DECSET_BIT_REVERSE_VIDEO = 1 << 1; @@ -185,8 +266,60 @@ public final class TerminalEmulator { /** Holds the bit flags which arguments are sub parameters (after a colon) - bit N is set if mArgs[N] is a sub parameter. */ private int mArgsSubParamsBitSet = 0; - /** Holds OSC and device control arguments, which can be strings. */ - private final StringBuilder mOSCOrDeviceControlArgs = new StringBuilder(); + + + /** + * The initial capacity for {@link #mTerminalControlArgs}. + */ + private static final int TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY = 16; + + /** + * The max length for {@link #mTerminalControlArgs}. + * Needs to be large enough to contain reasonable OSC 52 pastes, sixel and iterm images data. + */ + private static final int TERMINAL_CONTROL_ARGS__MAX_LENGTH = 16384; + + /** The terminal control arguments string buffer, like for OSC, DCS, APC commands. */ + private StringBuilder mTerminalControlArgs = new StringBuilder(TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY); + + + + /** + * The integer Operating System Command `type` received as `ESC ] type ;`. + * This will be set as soon as `type` followed by `;` is received, and before any further + * optional parameters are received. + */ + private int mOscType = -1; + + /** + * If `true`, then `processCodePoint()` will directly call `receiveOsc()` as a fast path + * without additional checks. + * + * Can be enabled for OSC commands via {@link #setOscTypeVariables()}. + */ + private boolean mIsFastPathOsc = false; + + /** + * If `true`, then `processCodePoint()` will not print any CR/LF characters received. + * This is ignored if `mIsFastPathOsc` is already `true` for a command. + * + * Can be enabled for OSC commands via {@link #setOscTypeVariables()}. + */ + private boolean mIgnoreCrLfForOsc = false; + + + /** + * If `true`, then `processCodePoint()` will directly call `doDcs()` as a fast path + * without additional checks. + * + * Can be enabled for DCS commands via {@link #doDcs(int)}. + */ + private boolean mIsFastPathDcs = false; + + /** Whether processing an `ESC` for a DCS command. */ + private boolean ESC_DCS__ESC = false; + + /** * True if the current escape sequence should continue, false if the current escape sequence should be terminated. @@ -359,6 +492,8 @@ private int getTerminalTranscriptRows(Integer transcriptRows) { return transcriptRows; } + + /** * @param mouseButton one of the MOUSE_* constants of this class. */ @@ -568,12 +703,34 @@ private void processByte(byte byteToProcess) { } public void processCodePoint(int b) { + if (mEscapeState == ESC_OSC && mIsFastPathOsc) { + mContinueSequence = false; + receiveOsc(b); + if (!mContinueSequence) mEscapeState = ESC_NONE; + return; + } + + if (mEscapeState == ESC_DCS && mIsFastPathDcs) { + if (b == 27) { // ESC + ESC_DCS__ESC = true; + return; + } + mContinueSequence = false; + doDcs(b); + if (!mContinueSequence) mEscapeState = ESC_NONE; + return; + } + // The Application Program-Control (APC) string might be arbitrary non-printable characters, so handle that early. if (mEscapeState == ESC_APC) { - doApc(b); + mContinueSequence = false; + receiveApc(b); + if (!mContinueSequence) mEscapeState = ESC_NONE; return; - } else if (mEscapeState == ESC_APC_ESCAPE) { - doApcEscape(b); + } else if (mEscapeState == ESC_APC__ESC) { + mContinueSequence = false; + receiveApcEsc(b); + if (!mContinueSequence) mEscapeState = ESC_NONE; return; } @@ -582,7 +739,7 @@ public void processCodePoint(int b) { break; case 7: // Bell (BEL, ^G, \a). If in an OSC sequence, BEL may terminate a string; otherwise signal bell. if (mEscapeState == ESC_OSC) - doOsc(b); + receiveOsc(b); else mSession.onBell(); break; @@ -632,13 +789,14 @@ public void processCodePoint(int b) { break; case 27: // ESC // Starts an escape sequence unless we're parsing a string - if (mEscapeState == ESC_P) { + if (mEscapeState == ESC_DCS) { // XXX: Ignore escape when reading device control sequence, since it may be part of string terminator. + ESC_DCS__ESC = true; return; } else if (mEscapeState != ESC_OSC) { startEscapeSequence(); } else { - doOsc(b); + receiveOsc(b); } break; default: @@ -838,13 +996,13 @@ public void processCodePoint(int b) { case ESC_PERCENT: break; case ESC_OSC: - doOsc(b); + receiveOsc(b); break; - case ESC_OSC_ESC: - doOscEsc(b); + case ESC_OSC__ESC: + receiveOscEsc(b); break; - case ESC_P: - doDeviceControl(b); + case ESC_DCS: + doDcs(b); break; case ESC_CSI_QUESTIONMARK_ARG_DOLLAR: if (b == 'p') { @@ -914,12 +1072,18 @@ public void processCodePoint(int b) { } } - /** When in {@link #ESC_P} ("device control") sequence. */ - private void doDeviceControl(int b) { - switch (b) { - case (byte) '\\': // End of ESC \ string Terminator - { - String dcs = mOSCOrDeviceControlArgs.toString(); + + + /** + * Do {@link #ESC_DCS}. Check its docs for more info. + */ + private void doDcs(final int b) { + if ( + // End of DCS if string terminator ST `ESC \` received. + (ESC_DCS__ESC && b == '\\') + ) { + String dcs = mTerminalControlArgs.toString(); + // DCS $ q P t ST. Request Status String (DECRQSS) if (dcs.startsWith("$q")) { if (dcs.equals("$q\"p")) { @@ -1022,43 +1186,21 @@ private void doDeviceControl(int b) { if (LOG_ESCAPE_SEQUENCES) Logger.logError(mClient, LOG_TAG, "Unrecognized device control string: " + dcs); } + + // Clear DCS args buffer and variables and finish sequence. + clearTerminalControlArgs(); + clearDcsTypeVariables(); finishSequence(); - } - break; - default: - if (mOSCOrDeviceControlArgs.length() > MAX_OSC_STRING_LENGTH) { - // Too long. - mOSCOrDeviceControlArgs.setLength(0); - finishSequence(); - } else { - mOSCOrDeviceControlArgs.appendCodePoint(b); - continueSequence(mEscapeState); - } - } - } + } else { + ESC_DCS__ESC = false; - /** - * When in {@link #ESC_APC} (APC, Application Program Command) sequence. - */ - private void doApc(int b) { - if (b == 27) { - continueSequence(ESC_APC_ESCAPE); + if (!collectTerminalControlArgs(b)) return; } - // Eat APC sequences silently for now. } - /** - * When in {@link #ESC_APC} (APC, Application Program Command) sequence. - */ - private void doApcEscape(int b) { - if (b == '\\') { - // A String Terminator (ST), ending the APC escape sequence. - finishSequence(); - } else { - // The Escape character was not the start of a String Terminator (ST), - // but instead just data inside of the APC escape sequence. - continueSequence(ESC_APC); - } + public void clearDcsTypeVariables() { + ESC_DCS__ESC = false; + mIsFastPathDcs = false; } private int nextTabStop(int numTabs) { @@ -1472,8 +1614,9 @@ private void doEsc(int b) { case '0': // SS3, ignore. break; case 'P': // Device control string - mOSCOrDeviceControlArgs.setLength(0); - continueSequence(ESC_P); + clearTerminalControlArgs(); + clearDcsTypeVariables(); + continueSequence(ESC_DCS); break; case '[': continueSequence(ESC_CSI); @@ -1482,13 +1625,15 @@ private void doEsc(int b) { setDecsetinternalBit(DECSET_BIT_APPLICATION_KEYPAD, true); break; case ']': // OSC - mOSCOrDeviceControlArgs.setLength(0); + clearTerminalControlArgs(); + clearOscTypeVariables(); continueSequence(ESC_OSC); break; case '>': // DECKPNM setDecsetinternalBit(DECSET_BIT_APPLICATION_KEYPAD, false); break; case '_': // APC - Application Program Command. + clearTerminalControlArgs(); continueSequence(ESC_APC); break; default: @@ -1981,44 +2126,154 @@ private void selectGraphicRendition() { } } - private void doOsc(int b) { + + + /** + * Receive {@link #ESC_APC}. Check its docs for more info. + */ + private void receiveApc(final int b) { + switch (b) { + case 27: // Escape. + continueSequence(ESC_APC__ESC); + break; + default: + if (!collectTerminalControlArgs(b)) return; + } + } + + /** + * Receive {@link #ESC_APC__ESC}. Check its docs for more info. + */ + private void receiveApcEsc(final int b) { + switch (b) { + case '\\': + //doApc(); + //clearApcTypeVariables(); + break; + default: + // The ESC character was not followed by a \, so insert the ESC and + // the current character in arg buffer. + if (!collectTerminalControlArgs(27)) return; + if (!collectTerminalControlArgs(b)) return; + continueSequence(ESC_APC); + break; + } + } + + /** + * Clear {@link #ESC_APC} type variables. + */ + public void clearApcTypeVariables() {} + + /** + * Do {@link #ESC_APC}. Check its docs for more info. + */ + private void doApc() {} + + + + + /** + * Receive {@link #ESC_OSC}. Check its docs for more info. + */ + private void receiveOsc(final int b) { switch (b) { case 7: // Bell. - doOscSetTextParameters("\007"); + doOsc("\007"); + clearOscTypeVariables(); break; case 27: // Escape. - continueSequence(ESC_OSC_ESC); + continueSequence(ESC_OSC__ESC); break; default: - collectOSCArgs(b); + if (!collectTerminalControlArgs(b)) return; + if (mOscType == -1) { + setOscTypeVariables(); + } break; } } - private void doOscEsc(int b) { + /** + * Receive {@link #ESC_OSC__ESC}. Check its docs for more info. + */ + private void receiveOscEsc(final int b) { switch (b) { case '\\': - doOscSetTextParameters("\033\\"); + doOsc("\033\\"); + clearOscTypeVariables(); break; default: // The ESC character was not followed by a \, so insert the ESC and // the current character in arg buffer. - collectOSCArgs(27); - collectOSCArgs(b); + if (!collectTerminalControlArgs(27)) return; + if (!collectTerminalControlArgs(b)) return; continueSequence(ESC_OSC); break; } } - /** An Operating System Controls (OSC) Set Text Parameters. May come here from BEL or ST. */ - private void doOscSetTextParameters(String bellOrStringTerminator) { + /** + * Set {@link #ESC_OSC} type variables. + */ + void setOscTypeVariables() { + if (mOscType >= 0) return; + if (mTerminalControlArgs.indexOf(":") < 0) return; + + int value = -1; + int argsLength = mTerminalControlArgs.length(); + + // Extract initial $value from initial "$value;..." string. + for (int i = 0; i < argsLength; i++) { + char b = mTerminalControlArgs.charAt(i); + if (b == ';') { + mOscType = value; + break; + } else if (b >= '0' && b <= '9') { + value = ((value < 0) ? 0 : value * 10) + (b - '0'); + } else { + mOscType = -2; // Unknown sequence. + return; + } + } + + if (mOscType >= 0) { + Integer terminalControlArgsCapacity = null; + switch (mOscType) { + } + + if (terminalControlArgsCapacity != null) { + ensureTerminalControlArgsCapacity(terminalControlArgsCapacity); + } + } + } + + /** + * Clear {@link #ESC_OSC} type variables. + */ + public void clearOscTypeVariables() { + mOscType = -1; + mIsFastPathOsc = false; + mIgnoreCrLfForOsc = false; + } + + /** + * Do {@link #ESC_OSC}. Check its docs for more info. + * + * This handles Set Text Parameters commands. + * + * The `bellOrStringTerminator` defines whether `OSC` command terminated with a `BEL` or `ST`. + */ + private void doOsc(String bellOrStringTerminator) { int value = -1; String textParameter = ""; + int argsLength = mTerminalControlArgs.length(); + // Extract initial $value from initial "$value;..." string. - for (int mOSCArgTokenizerIndex = 0; mOSCArgTokenizerIndex < mOSCOrDeviceControlArgs.length(); mOSCArgTokenizerIndex++) { - char b = mOSCOrDeviceControlArgs.charAt(mOSCArgTokenizerIndex); + for (int i = 0; i < argsLength; i++) { + char b = mTerminalControlArgs.charAt(i); if (b == ';') { - textParameter = mOSCOrDeviceControlArgs.substring(mOSCArgTokenizerIndex + 1); + textParameter = mTerminalControlArgs.substring(i + 1); break; } else if (b >= '0' && b <= '9') { value = ((value < 0) ? 0 : value * 10) + (b - '0'); @@ -2282,15 +2537,77 @@ private int getArg(int index, int defaultValue, boolean treatZeroAsDefault) { return result; } - private void collectOSCArgs(int b) { - if (mOSCOrDeviceControlArgs.length() < MAX_OSC_STRING_LENGTH) { - mOSCOrDeviceControlArgs.appendCodePoint(b); - continueSequence(mEscapeState); + + + /** Collect code point in {@link #mTerminalControlArgs}. */ + private boolean collectTerminalControlArgs(int b) { + if (mTerminalControlArgs.length() < TERMINAL_CONTROL_ARGS__MAX_LENGTH) { + try { + // Appending can cause an increase in capacity and cause an OOM. + mTerminalControlArgs.appendCodePoint(b); + continueSequence(mEscapeState); + return true; + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(mClient, LOG_TAG, "Terminal control args collect failed for" + + " char '" + (char) b + "' (numeric value=" + b + ") and" + + " args string '" + mTerminalControlArgs.substring(0, Math.min(16, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length() + + ": " + t.getMessage()); + } } else { - unknownSequence(b); + Logger.logError(mClient, LOG_TAG, "Terminal control args overflow for" + + " char '" + (char) b + "' (numeric value=" + b + ") and" + + " args string '" + mTerminalControlArgs.substring(0, Math.min(16, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length()); } + + clearTerminalControlArgs(); + finishSequence(); + return false; + } + + /** Clear {@link #mTerminalControlArgs}. */ + private void clearTerminalControlArgs() { + if (mTerminalControlArgs.capacity() <= TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY) { + // Mark existing buffer as empty and reuse old array already allocated in + // `StringBuffer` for future commands if required. + mTerminalControlArgs.setLength(0); + } else { + // `setLength()` will only update internal length marker and not reduce internal array + // capacity, and to deallocate extra memory `trimToSize()` needs to be called, which + // creates another smaller array. + // So just allocate a new object with an array with required initial capacity directly + // instead of setting length to 0, then trimming to create a smaller array, then + // increasing capacity again by creating a new array with required initial capacity by + // calling `ensureCapacity()`. + mTerminalControlArgs = new StringBuilder(TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY); + } + } + + /** + * Ensure enough capacity for {@link #mTerminalControlArgs} to prevent repeated reallocation of + * memory and copying as more data is received and appended, like with `append(char)`. + * + * The default capacity for {@link #mTerminalControlArgs} is defined by + * {@link #TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY}. + * + * By default, if `StringBuilder` reaches capacity, it sets new capacity to `(oldCapacity * 2) + 2`. + * So if initial capacity is `16`, and data to be received is 1024 bytes, then 6 reallocations + * will be done, so command processors should + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:libcore/ojluni/src/main/java/java/lang/AbstractStringBuilder.java;l=758 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:libcore/ojluni/src/main/java/java/lang/AbstractStringBuilder.java;l=183 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:libcore/ojluni/src/main/java/java/lang/AbstractStringBuilder.java;l=210 + * + * See also {@link StringBuilder#ensureCapacity(int)}. + * + * @param capacity The new capacity. + */ + private void ensureTerminalControlArgsCapacity(int capacity) { + mTerminalControlArgs.ensureCapacity(capacity); } + + + private void unimplementedSequence(int b) { logError("Unimplemented sequence char '" + (char) b + "' (U+" + String.format("%04x", b) + ")"); finishSequence(); @@ -2565,6 +2882,9 @@ public void reset() { mColors.reset(); mSession.onColorsChanged(); + + clearTerminalControlArgs(); + clearOscTypeVariables(); } public String getSelectedText(int x1, int y1, int x2, int y2) { From a7f2872c6ce93cdc1a10e5e5ca2aad9030b78f8d Mon Sep 17 00:00:00 2001 From: Matan Ziv-Av Date: Thu, 9 Apr 2026 03:39:44 +0500 Subject: [PATCH 07/15] Add sixel support: - In TerminalEmulator, interpret sixel sequences, and send them to TerminalBuffer for constructing a bitmap. - Sixel sequences may be longer than 8192 characters, so break them in natural places ($,-,#), rather than collecting all in the buffer. - The bitmap is sliced to character cell sized slices, and each the the style attribute is used to store which bitmap slice is displayed in place of this character. - In TerminalRenderer the style is interpreted, and drawn using drawBitmap, instead of drawText. Support iTerm inline image protocol (OSC 1337): - Using the same bitmap display infrastructure introduced for sixels. - Collects the image data outside of the OSC buffer. - Ignoring some parameters. Small emulator changes: - Also eat APC sequences, not echoing to screen. - Fix `CSI 14 t` to give actual size - Add `CSI 16 t` - Add `4` (sixel) to device attributes --- .../com/termux/terminal/TerminalBitmap.java | 187 +++++++++++++ .../com/termux/terminal/TerminalBuffer.java | 126 +++++++++ .../com/termux/terminal/TerminalEmulator.java | 256 +++++++++++++++++- .../java/com/termux/terminal/TerminalRow.java | 7 + .../java/com/termux/terminal/TextStyle.java | 22 ++ .../terminal/WorkingTerminalBitmap.java | 110 ++++++++ .../com/termux/view/TerminalRenderer.java | 26 +- 7 files changed, 729 insertions(+), 5 deletions(-) create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java new file mode 100644 index 0000000000..9758705bdc --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java @@ -0,0 +1,187 @@ +package com.termux.terminal; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Rect; + +import android.os.SystemClock; + +/** + * A circular buffer of {@link TerminalRow}:s which keeps notes about what is visible on a logical screen and the scroll + * history. + *

    + * See {@link #externalToInternalRow(int)} for how to map from logical screen rows to array indices. + */ +public class TerminalBitmap { + public Bitmap bitmap; + public int cellWidth; + public int cellHeight; + public int scrollLines; + public int[] cursorDelta; + private static final String LOG_TAG = "TerminalBitmap"; + + + public TerminalBitmap(int num, WorkingTerminalBitmap sixel, int Y, int X, int cellW, int cellH, TerminalBuffer screen) { + Bitmap bm = sixel.bitmap; + bm = resizeBitmapConstraints(bm, sixel.width, sixel.height, cellW, cellH, screen.mColumns - X); + addBitmap(num, bm, Y, X, cellW, cellH, screen); + } + + public TerminalBitmap(int num, byte[] image, int Y, int X, int cellW, int cellH, int width, int height, boolean aspect, TerminalBuffer screen) { + Bitmap bm = null; + int imageHeight; + int imageWidth; + int newWidth = width; + int newHeight = height; + if (height > 0 || width > 0) { + BitmapFactory.Options options = new BitmapFactory.Options(); + options.inJustDecodeBounds = true; + try { + BitmapFactory.decodeByteArray(image, 0, image.length, options); + } catch (Exception e) { + Logger.logWarn(null, LOG_TAG, "Cannot decode image"); + } + imageHeight = options.outHeight; + imageWidth = options.outWidth; + if (aspect) { + double wFactor = 9999.0; + double hFactor = 9999.0; + if (width > 0) { + wFactor = (double)width / imageWidth; + } + if (height > 0) { + hFactor = (double)height / imageHeight; + } + double factor = Math.min(wFactor, hFactor); + newWidth = (int)(factor * imageWidth); + newHeight = (int)(factor * imageHeight); + } else { + if (height <= 0) { + newHeight = imageHeight; + } + if (width <= 0) { + newWidth = imageWidth; + } + } + int scaleFactor = 1; + while (imageHeight >= 2 * newHeight * scaleFactor && imageWidth >= 2 * newWidth * scaleFactor) { + scaleFactor = scaleFactor * 2; + } + BitmapFactory.Options scaleOptions = new BitmapFactory.Options(); + scaleOptions.inSampleSize = scaleFactor; + try { + bm = BitmapFactory.decodeByteArray(image, 0, image.length, scaleOptions); + } catch (Exception e) { + Logger.logWarn(null, LOG_TAG, "Out of memory, cannot decode image"); + bitmap = null; + return; + } + if (bm == null) { + Logger.logWarn(null, LOG_TAG, "Could not decode image"); + bitmap = null; + return; + } + int maxWidth = (screen.mColumns - X) * cellW; + if (newWidth > maxWidth) { + int cropWidth = bm.getWidth() * maxWidth / newWidth; + try { + bm = Bitmap.createBitmap(bm, 0, 0, cropWidth, bm.getHeight()); + newWidth = maxWidth; + } catch(OutOfMemoryError e) { + // This is just a memory optimization. If it fails, + // continue (and probably fail later). + } + } + try { + bm = Bitmap.createScaledBitmap(bm, newWidth, newHeight, true); + } catch(OutOfMemoryError e) { + Logger.logWarn(null, LOG_TAG, "Out of memory, cannot rescale image"); + bm = null; + } + } else { + try { + bm = BitmapFactory.decodeByteArray(image, 0, image.length); + } catch (OutOfMemoryError e) { + Logger.logWarn(null, LOG_TAG, "Out of memory, cannot decode image"); + } + } + + if (bm == null) { + Logger.logWarn(null, LOG_TAG, "Cannot decode image"); + bitmap = null; + return; + } + + bm = resizeBitmapConstraints(bm, bm.getWidth(), bm.getHeight(), cellW, cellH, screen.mColumns - X); + addBitmap(num, bm, Y, X, cellW, cellH, screen); + cursorDelta = new int[] {scrollLines, (bitmap.getWidth() + cellW - 1) / cellW}; + } + + private void addBitmap(int num, Bitmap bm, int Y, int X, int cellW, int cellH, TerminalBuffer screen) { + if (bm == null) { + bitmap = null; + return; + } + int width = bm.getWidth(); + int height = bm.getHeight(); + cellWidth = cellW; + cellHeight = cellH; + int w = Math.min(screen.mColumns - X, (width + cellW - 1) / cellW); + int h = (height + cellH - 1) / cellH; + int s = 0; + for (int i=0; i cellW * Columns || (w % cellW) != 0 || (h % cellH) != 0) { + int newW = Math.min(cellW * Columns, ((w - 1) / cellW) * cellW + cellW); + int newH = ((h - 1) / cellH) * cellH + cellH; + try { + bm = resizeBitmap(bm, newW, newH); + } catch(OutOfMemoryError e) { + // Only a minor display glitch in this case + } + } + return bm; + } +} diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java index 796afa0ba4..8e8620d393 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java @@ -1,6 +1,15 @@ package com.termux.terminal; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.HashMap; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Rect; + +import android.os.SystemClock; /** * A circular buffer of {@link TerminalRow}:s which keeps notes about what is visible on a logical screen and the scroll @@ -20,6 +29,12 @@ public final class TerminalBuffer { /** The index in the circular buffer where the visible screen starts. */ private int mScreenFirstRow = 0; + public HashMap bitmaps; + public WorkingTerminalBitmap workingBitmap; + private boolean hasBitmaps; + private long bitmapLastGC; + + /** * Create a transcript screen. * @@ -35,6 +50,9 @@ public TerminalBuffer(int columns, int totalRows, int screenRows) { mLines = new TerminalRow[totalRows]; blockSet(0, 0, columns, screenRows, ' ', TextStyle.NORMAL); + hasBitmaps = false; + bitmaps = new HashMap(); + bitmapLastGC = SystemClock.uptimeMillis(); } public String getTranscriptText() { @@ -401,6 +419,28 @@ public void scrollDownOneLine(int topMargin, int bottomMargin, long style) { if (mLines[blankRow] == null) { mLines[blankRow] = new TerminalRow(mColumns, style); } else { + // find if a bitmap is completely scrolled out + Set used = new HashSet(); + if(mLines[blankRow].mHasBitmap) { + for (int column = 0; column < mColumns; column++) { + final long st = mLines[blankRow].getStyle(column); + if (TextStyle.isBitmap(st)) { + used.add((int)(st >> 16) & 0xffff); + } + } + TerminalRow nextLine = mLines[(blankRow + 1) % mTotalRows]; + if(nextLine.mHasBitmap) { + for (int column = 0; column < mColumns; column++) { + final long st = nextLine.getStyle(column); + if (TextStyle.isBitmap(st)) { + used.remove((int)(st >> 16) & 0xffff); + } + } + } + for(Integer bm: used) { + bitmaps.remove(bm); + } + } mLines[blankRow].clear(style); } } @@ -496,6 +536,92 @@ public void clearTranscript() { Arrays.fill(mLines, mScreenFirstRow - mActiveTranscriptRows, mScreenFirstRow, null); } mActiveTranscriptRows = 0; + bitmaps.clear(); + hasBitmaps = false; + } + + public Bitmap getSixelBitmap(int codePoint, long style) { + return bitmaps.get(TextStyle.bitmapNum(style)).bitmap; + } + + public Rect getSixelRect(int codePoint, long style ) { + TerminalBitmap bm = bitmaps.get(TextStyle.bitmapNum(style)); + int x = TextStyle.bitmapX(style); + int y = TextStyle.bitmapY(style); + Rect r = new Rect(x * bm.cellWidth, y * bm.cellHeight, (x+1) * bm.cellWidth, (y+1) * bm.cellHeight); + return r; + } + + public void sixelStart(int width, int height) { + workingBitmap = new WorkingTerminalBitmap(width, height); + } + + public void sixelChar(int c, int rep) { + workingBitmap.sixelChar(c, rep); + } + + public void sixelSetColor(int col) { + workingBitmap.sixelSetColor(col); + } + + public void sixelSetColor(int col, int r, int g, int b) { + workingBitmap.sixelSetColor(col, r, g, b); + } + + private int findFreeBitmap() { + int i = 0; + while (bitmaps.containsKey(i)) { + i++; + } + return i; + } + + public int sixelEnd(int Y, int X, int cellW, int cellH) { + int num = findFreeBitmap(); + bitmaps.put(num, new TerminalBitmap(num, workingBitmap, Y, X, cellW, cellH, this)); + workingBitmap = null; + if (bitmaps.get(num).bitmap == null) { + bitmaps.remove(num); + return 0; + } + hasBitmaps = true; + bitmapGC(30000); + return bitmaps.get(num).scrollLines; + } + + public int[] addImage(byte[] image, int Y, int X, int cellW, int cellH, int width, int height, boolean aspect) { + int num = findFreeBitmap(); + bitmaps.put(num, new TerminalBitmap(num, image, Y, X, cellW, cellH, width, height, aspect, this)); + if (bitmaps.get(num).bitmap == null) { + bitmaps.remove(num); + return new int[] {0,0}; + } + hasBitmaps = true; + bitmapGC(30000); + return bitmaps.get(num).cursorDelta; } + public void bitmapGC(int timeDelta) { + if (!hasBitmaps || bitmapLastGC + timeDelta > SystemClock.uptimeMillis()) { + return; + } + Set used = new HashSet(); + for (int line = 0; line < mLines.length; line++) { + if(mLines[line] != null && mLines[line].mHasBitmap) { + for (int column = 0; column < mColumns; column++) { + final long st = mLines[line].getStyle(column); + if (TextStyle.isBitmap(st)) { + used.add((int)(st >> 16) & 0xffff); + } + } + } + } + Set keys = new HashSet(bitmaps.keySet()); + for (Integer bn: keys) { + if (!used.contains(bn)) { + bitmaps.remove(bn); + } + } + bitmapLastGC = SystemClock.uptimeMillis(); + } } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index 0eefd5d01e..1fe43023bc 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -3,6 +3,7 @@ import android.util.Base64; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; import java.util.Objects; @@ -330,6 +331,12 @@ public final class TerminalEmulator { /** The current state of the escape sequence state machine. One of the ESC_* constants. */ private int mEscapeState; + private boolean ESC_DCS_escape = false; + private boolean ESC_DCS_sixel = false; + private ArrayList ESC_OSC_data; + private int ESC_OSC_colon = 0; + private boolean ESC_OSC_outofmem = false; + private final SavedScreenState mSavedStateMain = new SavedScreenState(); private final SavedScreenState mSavedStateAlt = new SavedScreenState(); @@ -406,6 +413,13 @@ public final class TerminalEmulator { private static final String LOG_TAG = "TerminalEmulator"; + private int cellW = 12, cellH = 12; + + public void setCellSize(int w, int h) { + cellW = w; + cellH = h; + } + private boolean isDecsetInternalBitSet(int bit) { return (mCurrentDecSetFlags & bit) != 0; } @@ -734,6 +748,7 @@ public void processCodePoint(int b) { return; } + mScreen.bitmapGC(300000); switch (b) { case 0: // Null character (NUL, ^@). Do nothing. break; @@ -768,10 +783,16 @@ public void processCodePoint(int b) { case 10: // Line feed (LF, \n). case 11: // Vertical tab (VT, \v). case 12: // Form feed (FF, \f). - doLinefeed(); + if((mEscapeState != ESC_DCS || !ESC_DCS_sixel) && ESC_OSC_colon <= 0) { + // Ignore CR/LF inside sixels or iterm2 data + doLinefeed(); + } break; case 13: // Carriage return (CR, \r). - setCursorCol(mLeftMargin); + if((mEscapeState != ESC_DCS || !ESC_DCS_sixel) && ESC_OSC_colon <= 0) { + // Ignore CR/LF inside sixels or iterm2 data + setCursorCol(mLeftMargin); + } break; case 14: // Shift Out (Ctrl-N, SO) → Switch to Alternate Character Set. This invokes the G1 character set. mUseLineDrawingUsesG0 = false; @@ -1078,9 +1099,20 @@ public void processCodePoint(int b) { * Do {@link #ESC_DCS}. Check its docs for more info. */ private void doDcs(final int b) { + boolean firstSixel = false; + if (!ESC_DCS_sixel && (b=='$' || b=='-' || b=='#')) { + //Check if sixel sequence that needs breaking + String dcs = mTerminalControlArgs.toString(); + if (dcs.matches("[0-9;]*q.*")) { + firstSixel = true; + } + } + if ( // End of DCS if string terminator ST `ESC \` received. (ESC_DCS__ESC && b == '\\') + // Sixel sequences may be very long. '$' and '!' are natural for breaking the sequence. + || firstSixel || (ESC_DCS_sixel && (b=='$' || b=='-' || b=='#')) ) { String dcs = mTerminalControlArgs.toString(); @@ -1182,6 +1214,95 @@ private void doDcs(final int b) { Logger.logError(mClient, LOG_TAG, "Invalid device termcap/terminfo name of odd length: " + part); } } + } else if (ESC_DCS_sixel || dcs.matches("[0-9;]*q.*")) { + int pos = 0; + if (!ESC_DCS_sixel) { + ESC_DCS_sixel = true; + mScreen.sixelStart(100, 100); + while (dcs.codePointAt(pos) != 'q') { + pos++; + } + pos++; + } + if (b=='$' || b=='-') { + // Add to string + dcs = dcs + (char)b; + } + int rep = 1; + while (pos < dcs.length()) { + if (dcs.codePointAt(pos) == '"') { + pos++; + int args[]={0,0,0,0}; + int arg = 0; + while (pos < dcs.length() && ((dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') || dcs.codePointAt(pos) == ';')) { + if (dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { + args[arg] = args[arg] * 10 + dcs.codePointAt(pos) - '0'; + } else { + arg++; + if (arg > 3) { + break; + } + } + pos++; + } + if (pos == dcs.length()) { + break; + } + } else if (dcs.codePointAt(pos) == '#') { + int col = 0; + pos++; + while (pos < dcs.length() && dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { + col = col * 10 + dcs.codePointAt(pos++) - '0'; + } + if (pos == dcs.length() || dcs.codePointAt(pos) != ';') { + mScreen.sixelSetColor(col); + } else { + pos++; + int args[]={0,0,0,0}; + int arg = 0; + while (pos < dcs.length() && ((dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') || dcs.codePointAt(pos) == ';')) { + if (dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { + args[arg] = args[arg] * 10 + dcs.codePointAt(pos) - '0'; + } else { + arg++; + if (arg > 3) { + break; + } + } + pos++; + } + if (args[0] == 2) { + mScreen.sixelSetColor(col, args[1], args[2], args[3]); + } + } + } else if (dcs.codePointAt(pos) == '!') { + rep = 0; + pos++; + while (pos < dcs.length() && dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { + rep = rep * 10 + dcs.codePointAt(pos++) - '0'; + } + } else if (dcs.codePointAt(pos) == '$' || dcs.codePointAt(pos) == '-' || (dcs.codePointAt(pos) >= '?' && dcs.codePointAt(pos) <= '~')) { + mScreen.sixelChar(dcs.codePointAt(pos++), rep); + rep = 1; + } else { + pos++; + } + } + if (b == '\\') { + ESC_DCS_sixel = false; + int n = mScreen.sixelEnd(mCursorRow, mCursorCol, cellW, cellH); + for(;n>0;n--) { + doLinefeed(); + } + } else { + mTerminalControlArgs.setLength(0); + if (b=='#') { + mTerminalControlArgs.appendCodePoint(b); + } + // Do not finish sequence + continueSequence(mEscapeState); + return; + } } else { if (LOG_ESCAPE_SEQUENCES) Logger.logError(mClient, LOG_TAG, "Unrecognized device control string: " + dcs); @@ -1628,6 +1749,7 @@ private void doEsc(int b) { clearTerminalControlArgs(); clearOscTypeVariables(); continueSequence(ESC_OSC); + ESC_OSC_colon = -1; break; case '>': // DECKPNM setDecsetinternalBit(DECSET_BIT_APPLICATION_KEYPAD, false); @@ -1862,7 +1984,7 @@ private void doCsi(int b) { // The important part that may still be used by some (tmux stores this value but does not currently use it) // is the first response parameter identifying the terminal service class, where we send 64 for "vt420". // This is followed by a list of attributes which is probably unused by applications. Send like xterm. - if (getArg0(0) == 0) mSession.write("\033[?64;1;2;6;9;15;18;21;22c"); + if (getArg0(0) == 0) mSession.write("\033[?64;1;2;4;6;9;15;18;21;22c"); break; case 'd': // ESC [ Pn d - Vert Position Absolute setCursorRow(Math.min(Math.max(1, getArg0(1)), mRows) - 1); @@ -2190,6 +2312,29 @@ private void receiveOsc(final int b) { if (mOscType == -1) { setOscTypeVariables(); } + + if (ESC_OSC_colon == -1 && b == ':') { + // Collect base64 data for OSC 1337 + ESC_OSC_colon = mTerminalControlArgs.length(); + ESC_OSC_data = new ArrayList(65536); + ESC_OSC_outofmem = false; + } else if (ESC_OSC_colon >= 0 && mTerminalControlArgs.length() - ESC_OSC_colon == 4) { + if (!ESC_OSC_outofmem) { + try { + byte[] decoded = Base64.decode(mTerminalControlArgs.substring(ESC_OSC_colon), 0); + for (int i = 0 ; i < decoded.length; i++) { + ESC_OSC_data.add(decoded[i]); + } + } catch(Exception e) { + // Ignore non-Base64 data. + } catch(OutOfMemoryError e) { + // Out of memory + // Keep decoding, but fo not collect the data + ESC_OSC_outofmem = true; + } + } + mTerminalControlArgs.setLength(ESC_OSC_colon); + } break; } } @@ -2266,6 +2411,8 @@ public void clearOscTypeVariables() { */ private void doOsc(String bellOrStringTerminator) { int value = -1; + int osc_colon = ESC_OSC_colon; + ESC_OSC_colon = -1; String textParameter = ""; int argsLength = mTerminalControlArgs.length(); @@ -2403,6 +2550,105 @@ private void doOsc(String bellOrStringTerminator) { break; case 119: // Reset highlight color. break; + case 1337: // iTerm extemsions + if (textParameter.startsWith("File=")) { + int pos = 5; + boolean inline = false; + boolean aspect = true; + int width = -1; + int height = -1; + while (pos < textParameter.length()) { + int eqpos = textParameter.indexOf('=', pos); + if (eqpos == -1) { + break; + } + int semicolonpos = textParameter.indexOf(';', eqpos); + if (semicolonpos == -1) { + semicolonpos = textParameter.length() - 1; + } + String k = textParameter.substring(pos, eqpos); + String v = textParameter.substring(eqpos + 1, semicolonpos); + pos = semicolonpos + 1; + if (k.equalsIgnoreCase("inline")) { + inline = v.equals("1"); + } + if (k.equalsIgnoreCase("preserveAspectRatio")) { + aspect = ! v.equals("0"); + } + if (k.equalsIgnoreCase("width")) { + double factor = cellW; + int div = 1; + int e = v.length(); + if (v.endsWith("px")) { + factor = 1; + e -= 2; + } else if (v.endsWith("%")) { + factor = 0.01 * cellW * mColumns; + e -= 1; + } + try { + width = (int)(factor * Integer.parseInt(v.substring(0,e))); + } catch(Exception ex) { + } + } + if (k.equalsIgnoreCase("height")) { + double factor = cellH; + int div = 1; + int e = v.length(); + if (v.endsWith("px")) { + factor = 1; + e -= 2; + } else if (v.endsWith("%")) { + factor = 0.01 * cellH * mRows; + e -= 1; + } + try { + height = (int)(factor * Integer.parseInt(v.substring(0,e))); + } catch(Exception ex) { + } + } + } + if (!inline) { + finishSequence(); + return; + } + if (osc_colon >= 0 && mTerminalControlArgs.length() > osc_colon) { + while (mTerminalControlArgs.length() - osc_colon < 4) { + mTerminalControlArgs.append('='); + } + try { + byte[] decoded = Base64.decode(mTerminalControlArgs.substring(osc_colon), 0); + for (int i = 0 ; i < decoded.length; i++) { + ESC_OSC_data.add(decoded[i]); + } + } catch(Exception e) { + // Ignore non-Base64 data. + } + mTerminalControlArgs.setLength(osc_colon); + } + if (osc_colon >= 0) { + byte[] result = new byte[ESC_OSC_data.size()]; + for(int i = 0; i < ESC_OSC_data.size(); i++) { + result[i] = ESC_OSC_data.get(i).byteValue(); + } + int[] res = mScreen.addImage(result, mCursorRow, mCursorCol, cellW, cellH, width, height, aspect); + int col = res[1] + mCursorCol; + if (col < mColumns -1) { + res[0] -= 1; + } else { + col = 0; + } + for(;res[0] > 0; res[0]--) { + doLinefeed(); + } + mCursorCol = col; + ESC_OSC_data.clear(); + } else { + } + } else if (textParameter.startsWith("ReportCellSize")) { + mSession.write(String.format(Locale.US, "\0331337;ReportCellSize=%d;%d\007", cellH, cellW)); + } + break; default: unknownParameter(value); break; @@ -2883,6 +3129,10 @@ public void reset() { mColors.reset(); mSession.onColorsChanged(); + ESC_DCS_escape = false; + ESC_DCS_sixel = false; + ESC_OSC_colon = -1; + clearTerminalControlArgs(); clearOscTypeVariables(); } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java index 8d7e6bab20..a70103482f 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java @@ -49,6 +49,8 @@ public final class TerminalRow { final long[] mStyle; /** If this row might contain chars with width != 1, used for deactivating fast path */ boolean mHasNonOneWidthOrSurrogateChars; + /** If this row has a bitmap. Used for performace only */ + public boolean mHasBitmap; /** Construct a blank row (containing only whitespace, ' ') with a specified style. */ public TerminalRow(int columns, long style) { @@ -146,6 +148,7 @@ public void clear(long style) { Arrays.fill(mStyle, style); mSpaceUsed = mColumns; mHasNonOneWidthOrSurrogateChars = false; + mHasBitmap = false; } // https://github.com/steven676/Android-Terminal-Emulator/commit/9a47042620bec87617f0b4f5d50568535668fe26 @@ -155,6 +158,10 @@ public void setChar(int columnToSet, int codePoint, long style) { mStyle[columnToSet] = style; + if (!mHasBitmap && TextStyle.isBitmap(style)) { + mHasBitmap = true; + } + final int newCodePointDisplayWidth = WcWidth.width(codePoint); // Fast path when we don't have any chars with width != 1 diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java b/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java index 173d6ae94e..7ee4b06ebc 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java @@ -35,6 +35,8 @@ public final class TextStyle { private final static int CHARACTER_ATTRIBUTE_TRUECOLOR_FOREGROUND = 1 << 9; /** If true (24-bit) color is used for the cell for foreground. */ private final static int CHARACTER_ATTRIBUTE_TRUECOLOR_BACKGROUND= 1 << 10; + /** If true, character represents a bitmap slice, not text. */ + public final static int BITMAP = 1 << 15; public final static int COLOR_INDEX_FOREGROUND = 256; public final static int COLOR_INDEX_BACKGROUND = 257; @@ -87,4 +89,24 @@ public static int decodeEffect(long style) { return (int) (style & 0b11111111111); } + public static long encodeBitmap(int num, int X, int Y) { + return ((long)num << 16) | ((long)Y << 32) | ((long)X << 48) | BITMAP; + } + + public static boolean isBitmap(long style) { + return (style & 0x8000) != 0; + } + + public static int bitmapNum(long style) { + return (int)(style & 0xffff0000) >> 16; + } + + public static int bitmapX(long style) { + return (int)((style >> 48) & 0xfff); + } + + public static int bitmapY(long style) { + return (int)((style >> 32) & 0xfff); + } + } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java new file mode 100644 index 0000000000..e9f0e2ed05 --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java @@ -0,0 +1,110 @@ +package com.termux.terminal; + +import android.graphics.Bitmap; + +/** + * A circular buffer of {@link TerminalRow}:s which keeps notes about what is visible on a logical screen and the scroll + * history. + *

    + * See {@link #externalToInternalRow(int)} for how to map from logical screen rows to array indices. + */ +public final class WorkingTerminalBitmap { + final private int sixelInitialColorMap[] = {0xFF000000, 0xFF3333CC, 0xFFCC2323, 0xFF33CC33, 0xFFCC33CC, 0xFF33CCCC, 0xFFCCCC33, 0xFF777777, + 0xFF444444, 0xFF565699, 0xFF994444, 0xFF569956, 0xFF995699, 0xFF569999, 0xFF999956, 0xFFCCCCCC}; + private int[] colorMap; + private int curX; + private int curY; + private int color; + public int width; + public int height; + public Bitmap bitmap; + private static final String LOG_TAG = "WorkingTerminalBitmap"; + + public WorkingTerminalBitmap(int w, int h) { + try { + bitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + } catch (OutOfMemoryError e) { + Logger.logWarn(null, LOG_TAG, "Out of memory - sixel ignored"); + bitmap = null; + } + bitmap.eraseColor(0); + width = 0; + height = 0; + curX = 0; + curY = 0; + colorMap = new int[256]; + for (int i=0; i<16; i++) { + colorMap[i] = sixelInitialColorMap[i]; + } + color = colorMap[0]; + } + + public void sixelChar(int c, int rep) { + if (bitmap == null) { + return; + } + if (c == '$') { + curX = 0; + return; + } + if (c == '-') { + curX = 0; + curY += 6; + return; + } + if (bitmap.getWidth() < curX + rep) { + try { + bitmap = TerminalBitmap.resizeBitmap(bitmap, curX + rep + 100, bitmap.getHeight()); + } catch(OutOfMemoryError e) { + Logger.logWarn(null, LOG_TAG, "Out of memory - sixel truncated"); + } + } + if (bitmap.getHeight() < curY + 6) { + // Very unlikely to resize both at the same time + try { + bitmap = TerminalBitmap.resizeBitmap(bitmap, bitmap.getWidth(), curY + 100); + } catch(OutOfMemoryError e) { + Logger.logWarn(null, LOG_TAG, "Out of memory - sixel truncated"); + } + } + if (curX + rep > bitmap.getWidth()) { + rep = bitmap.getWidth() - curX; + } + if ( curY + 6 > bitmap.getHeight()) { + return; + } + if (rep > 0 && c >= '?' && c <= '~') { + int b = c - '?'; + if (curY + 6 > height) { + height = curY + 6; + } + while (rep-- > 0) { + for (int i = 0 ; i < 6 ; i++) { + if ((b & (1< width) { + width = curX; + } + } + } + } + + public void sixelSetColor(int col) { + if (col >= 0 && col < 256) { + color = colorMap[col]; + } + } + + public void sixelSetColor(int col, int r, int g, int b) { + if (col >= 0 && col < 256) { + int red = Math.min(255, r*255/100); + int green = Math.min(255, g*255/100); + int blue = Math.min(255, b*255/100); + color = 0xff000000 + (red << 16) + (green << 8) + blue; + colorMap[col] = color; + } + } +} diff --git a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java index 214df331b3..2be86f17fa 100644 --- a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java +++ b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java @@ -1,8 +1,11 @@ package com.termux.view; +import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.Paint; import android.graphics.PorterDuff; +import android.graphics.Rect; +import android.graphics.RectF; import android.graphics.Typeface; import android.os.Build; @@ -66,6 +69,7 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final TerminalBuffer screen = mEmulator.getScreen(); final int[] palette = mEmulator.mColors.mCurrentColors; final int cursorShape = mEmulator.getCursorStyle(); + mEmulator.setCellSize((int)mFontWidth, (int)mFontLineSpacing); if (reverseVideo) canvas.drawColor(palette[TextStyle.COLOR_INDEX_FOREGROUND], PorterDuff.Mode.SRC); @@ -99,10 +103,28 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final boolean charIsHighsurrogate = Character.isHighSurrogate(charAtIndex); final int charsForCodePoint = charIsHighsurrogate ? 2 : 1; final int codePoint = charIsHighsurrogate ? Character.toCodePoint(charAtIndex, line[currentCharIndex + 1]) : charAtIndex; + final long style = lineObject.getStyle(column); + if (TextStyle.isBitmap(style)) { + Bitmap bm = mEmulator.getScreen().getSixelBitmap(codePoint, style); + if (bm != null) { + float left = column * mFontWidth; + float top = heightOffset - mFontLineSpacing; + RectF r = new RectF(left, top, left + mFontWidth, top + mFontLineSpacing); + canvas.drawBitmap(mEmulator.getScreen().getSixelBitmap(codePoint, style), mEmulator.getScreen().getSixelRect(codePoint, style), r, null); + } + column += 1; + measuredWidthForRun = 0.f; + lastRunStyle = 0; + lastRunInsideCursor = false; + lastRunStartColumn = column + 1; + lastRunStartIndex = currentCharIndex; + lastRunFontWidthMismatch = false; + currentCharIndex += charsForCodePoint; + continue; + } final int codePointWcWidth = WcWidth.width(codePoint); final boolean insideCursor = (cursorX == column || (codePointWcWidth == 2 && cursorX == column + 1)); final boolean insideSelection = column >= selx1 && column <= selx2; - final long style = lineObject.getStyle(column); // Check if the measured text width for this code point is not the same as that expected by wcwidth(). // This could happen for some fonts which are not truly monospace, or for more exotic characters such as @@ -113,7 +135,7 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final boolean fontWidthMismatch = Math.abs(measuredCodePointWidth / mFontWidth - codePointWcWidth) > 0.01; if (style != lastRunStyle || insideCursor != lastRunInsideCursor || insideSelection != lastRunInsideSelection || fontWidthMismatch || lastRunFontWidthMismatch) { - if (column == 0) { + if (column == 0 || column == lastRunStartColumn) { // Skip first column as there is nothing to draw, just record the current style. } else { final int columnWidthSinceLastRun = column - lastRunStartColumn; From 69ce502d8b17d6103f2cd6276dfe21ba9d1d9778 Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Thu, 9 Apr 2026 15:41:29 +0500 Subject: [PATCH 08/15] Added: Add terminal support for Bitmaps, Sixel (`DCS q`) and iTerm Image (`OSC 1337`) Support for displaying bitmaps inside the terminal has been added via `TerminalBitmap` which can be created from image `byte[]` or sixel bitmap. The bitmaps are sliced to character cell sized slices. The `TerminalBuffer` stores a map for bitmap number to the `TerminalBitmap` loaded in the terminal. The bitmap number and coordinates are encoded in the `long` `TerminalRow.mStyle` for the `TerminalRow` character of a column by `TerminalBitmap#buildOrThrow()` by getting encoded value from `TextStyle.encodeTerminalBitmap()`. The `TerminalRenderer.render()` then checks during rendering terminal output whether a character at a row/coloumn index is a bitmap instead of text by calling `TextStyle.isTerminalBitmap()`, then draws it using `Canvas.drawBitmap()` instead of `Canvas.drawText()`. Sixel images can be created with Sixel Device Control String command sent via `DCS q s..s ST` or `DCS P1; P2; P3; q s..s ST`. - The `TerminalEmulator` interprets sixel sequences, and sends them to `TerminalBuffer` for constructing a `TerminalSixel`. Once the sixel command has been completely processed, a `TerminalBitmap` is created from the `TerminalSixel`. If an error occurred during processing (like OOM), then remaining sixel command is completely read, but is ignored and no sixel is drawn (done by setting `mTerminalSixel` to `null` so that `TerminalBuffer.sixelReadData()` ignores further commands). - Since a sixel sequence can be very long to render a full image and can have length greater than `TERMINAL_CONTROL_ARGS__MAX_LENGTH` (`16384`), the entire sequence is not stored in the `mTerminalControlArgs` buffer before being processed as it will result in an overflow error, instead as soon as length crosses `TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2` and a complete sixel sub command (`#`, `!`, or `"`) has been received, it is immediately processed, and then further commands are read after emptying buffer. - If "rough" horizontal and vertical size of image is received at start of sixel data string with a `Raster Attributes` command, like done by `img2sixel` command, then sixel commands args buffer capacity (`mTerminalControlArgs`) is increased and sixel bitmap in `TerminalSixel` is resized at start, instead of having to keep resizing buffer/bitmap as more sixel data is received, which has a performance hit due to memory reallocations and copying. - The `4` (sixel) value has been added to `CSI` `Primary Device Attributes` terminal response. The `img2sixel` command can be used to display sixel images after installing with `libsixel` package with `pkg install libsixel`, like with `img2sixel --width=1000px image.jpg`. To manually send an escape sequence, check the `digiater.nl` link below, but it is too cumbersome to create images large enough to be easy viewable in the terminal. See Also: - https://vt100.net/docs/vt3xx-gp/chapter14.html - https://en.wikipedia.org/wiki/Sixel - https://www.digiater.nl/openvms/decus/vax90b1/krypton-nasa/all-about-sixels.text iTerm images can be created with `1337` Operating System Control command. - Both `File=` and `MultipartFile=` (chunk based) protocols are supported. The `inline` parameter should be `1` to display inline images in the terminal. Downloading images to Downloads folder with the value `0` will be ignored as that is not supported. - The escape sequences/image data cannot be greater than `TERMINAL_CONTROL_ARGS__MAX_LENGTH` (`16384`) bytes if sent via (`File=`) protocol, otherwise it will be ignored with an overflow error. For larger images, send images via `MultipartFile=` protocol in chunks with `FilePart=`, the `imgcat` utility uses that with 200-byte chunks if `--legacy` flag is not passed. - The `TerminalEmulator` interprets iTerm images sequences and creates an `ITermImage` to process parameters and store the base64 encoded image sent. Once all the data has been received, which can be over multiple `OSC` commands for `MultipartFile=` protocol, the encoded image is decoded to a `byte[]`, which is then passed to `TerminalBuffer`, which creates a `TerminalBitmap` for the image. The `imgcat` utility can be used for sending images, like with `imgcat --width 1000px image.jpg` (`MultipartFile=`) or `imgcat --width 1000px --legacy image.jpg` (`File=`). To manually send an escape sequence, run `echo -en '\e]1337;File=inline=1;keepAspectRatio=0;width=1000px;:' ; base64 -w 0 ./image.jpg ; echo -e '\e\\'` (`File=`). See Also: - https://iterm2.com/documentation-images.html - https://iterm2.com/utilities/imgcat - https://github.com/gnachman/iTerm2-shell-integration/blob/d1d4012068c3c6761d5676c28ed73e0e2df2b715/utilities/imgcat --- .../java/com/termux/terminal/ITermImage.java | 399 +++++++++ .../com/termux/terminal/TerminalBitmap.java | 458 +++++++---- .../com/termux/terminal/TerminalBuffer.java | 304 +++++-- .../com/termux/terminal/TerminalEmulator.java | 761 ++++++++++++------ .../java/com/termux/terminal/TerminalRow.java | 10 +- .../com/termux/terminal/TerminalSixel.java | 289 +++++++ .../java/com/termux/terminal/TextStyle.java | 30 +- .../terminal/WorkingTerminalBitmap.java | 110 --- .../com/termux/view/TerminalRenderer.java | 12 +- 9 files changed, 1775 insertions(+), 598 deletions(-) create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/ITermImage.java create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java delete mode 100644 terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java diff --git a/terminal-emulator/src/main/java/com/termux/terminal/ITermImage.java b/terminal-emulator/src/main/java/com/termux/terminal/ITermImage.java new file mode 100644 index 0000000000..b8da113c6d --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/ITermImage.java @@ -0,0 +1,399 @@ +package com.termux.terminal; + +import android.util.Base64; + +import java.util.Arrays; + +/** + * An iTerm image received via `OSC 1337`. + * + * - https://iterm2.com/documentation-images.html + */ +public class ITermImage { + + public static final String LOG_TAG = "ITermImage"; + + + + /** The {@link Enum} that defines {@link ITermImage} state. */ + public enum ImageState { + + INIT("init", 0), + ARGUMENTS_READ("arguments_read", 1), + IMAGE_READING("image_reading", 2), + IMAGE_READ("image_read", 3), + IMAGE_DECODED("image_decoded", 4), + FAILED("Failed", 5); + + private final String name; + private final int value; + + ImageState(final String name, final int value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public int getValue() { + return value; + } + + } + + + + protected final TerminalSessionClient mClient; + + protected final boolean mIsMultipart; + + protected int mWidth = -1; + protected int mHeight = -1; + + protected boolean mInline = false; + + protected boolean mPreserveAspectRatio = true; + + protected final StringBuilder mEncodedImage = new StringBuilder(/* Initial capacity. */ 4096); + protected byte[] mDecodedImage; + + /** The current state of the {@link ImageState}. */ + protected ImageState mCurrentState = ImageState.INIT; + /** The previous state of the {@link ImageState}. */ + protected ImageState mPreviousState = ImageState.INIT; + + + + protected ITermImage(TerminalSessionClient client, boolean isMultiPart) { + mClient = client; + + mIsMultipart = isMultiPart; + } + + + + public TerminalSessionClient getClient() { + return mClient; + } + + + public boolean isMultipart() { + return mIsMultipart; + } + + + public int getWidth() { + return mWidth; + } + + public int getHeight() { + return mHeight; + } + + + public boolean isInline() { + return mInline; + } + + + public boolean shouldPreserveAspectRatio() { + return mPreserveAspectRatio; + } + + + public String getEncodedImage() { + return mEncodedImage.toString(); + } + + public byte[] getDecodedImage() { + return mDecodedImage; + } + + + public synchronized ImageState getCurrentState() { + return mCurrentState; + } + + public synchronized ImageState getPreviousState() { + return mPreviousState; + } + + + protected synchronized boolean setState(ImageState newState) { + // The state transition cannot go back or change if already at `ImageState.IMAGE_DECODED` + if (newState.getValue() < mCurrentState.getValue() || mCurrentState == ImageState.IMAGE_DECODED) { + Logger.logError(mClient, LOG_TAG, + "Invalid image state transition from \"" + mCurrentState.getName() + "\" to " + "\"" + newState.getName() + "\""); + return false; + } + + // The `ImageState.FAILED` can be set again, like to add more errors, but we don't update + // `mPreviousState` with the `mCurrentState` value if its at `ImageState.FAILED` to + // preserve the last valid state. + if (mCurrentState != ImageState.FAILED) + mPreviousState = mCurrentState; + + mCurrentState = newState; + return true; + } + + + protected synchronized boolean setStateFailed(String error) { + if (error != null) { + Logger.logError(mClient, LOG_TAG, error); + } + return setState(ImageState.FAILED); + } + + + protected synchronized boolean ensureState(ImageState expectedState) { + return ensureState(expectedState, null); + } + + protected synchronized boolean ensureState(ImageState expectedState, String functionName) { + if (mCurrentState != expectedState) { + Logger.logError(mClient, LOG_TAG, + "The current image state is \"" + mCurrentState.getName() + "\" but expected \"" + expectedState.getName() + "\"" + + (functionName != null ? " while calling '" + functionName : "'") + + " for " + (!mIsMultipart ? "singlepart" : "multipart") + " image"); + return false; + } + return true; + } + + + public synchronized boolean isArgumentsRead() { + return mCurrentState == ImageState.ARGUMENTS_READ; + } + + public synchronized boolean isImageReading() { + return mCurrentState == ImageState.IMAGE_READING; + } + + public synchronized boolean isImageRead() { + return mCurrentState == ImageState.IMAGE_READ; + } + + public synchronized boolean isImageDecoded() { + return mCurrentState == ImageState.IMAGE_DECODED; + } + + + + public synchronized int readArguments(TerminalEmulator terminalEmulator, StringBuilder oscArgs, int index) { + if (!ensureState(ImageState.INIT, "ImageState.readArguments()")) { + return -1; + } + + boolean lastParam = false; + while (index < oscArgs.length()) { + char ch = oscArgs.charAt(index); + // End of optional arguments. + if (ch == ':' && !mIsMultipart) { + break; + } else if (ch == ' ') { + index++; + continue; + } + + int keyEndIndex = oscArgs.indexOf("=", index); + if (keyEndIndex == -1) { + setStateFailed("The key for an argument not found after index " + index + " in osc argument string: " + oscArgs); + return -1; + } + String argKey = oscArgs.substring(index, keyEndIndex); + + int valueEndIndex = oscArgs.indexOf(";", keyEndIndex); + if (valueEndIndex == -1) { + if (!mIsMultipart) { + // The last key value for `File=` command arguments may end with a colon `:` instead of a semi colon `;`. + valueEndIndex = oscArgs.indexOf(":", keyEndIndex); + if (valueEndIndex == -1) { + setStateFailed("The value for an argument not found after index " + index + " in osc argument string: " + oscArgs); + return -1; + } else { + index = valueEndIndex; + lastParam = true; + } + } else { + // The last key value for `MultipartFile=` command arguments may end without a semi colon `;`. + valueEndIndex = oscArgs.length(); + index = valueEndIndex; + } + } else { + index = valueEndIndex + 1; + } + + if (valueEndIndex <= keyEndIndex) { + setStateFailed("The argument key end index " + keyEndIndex + + " is <= value end index " + valueEndIndex + " in osc argument string: " + oscArgs); + return -1; + } + + String argValue = oscArgs.substring(keyEndIndex + 1, valueEndIndex); + + if (argKey.equalsIgnoreCase("inline")) { + mInline = argValue.equals("1"); + } + else if (argKey.equalsIgnoreCase("preserveAspectRatio")) { + mPreserveAspectRatio = !argValue.equals("0"); + } + else if (argKey.equalsIgnoreCase("width")) { + double factor = terminalEmulator.getCellWidthPixels(); + int intValueEndIndex = argValue.length(); + if (argValue.endsWith("px")) { + factor = 1; + intValueEndIndex -= 2; + } else if (argValue.endsWith("%")) { + factor = 0.01 * terminalEmulator.getCellWidthPixels() * terminalEmulator.getColumns(); + intValueEndIndex -= 1; + } + try { + mWidth = (int) (factor * Integer.parseInt(argValue.substring(0, intValueEndIndex))); + } catch (Exception e) { + } + } + else if (argKey.equalsIgnoreCase("height")) { + double factor = terminalEmulator.getCellHeightPixels(); + int intValueEndIndex = argValue.length(); + if (argValue.endsWith("px")) { + factor = 1; + intValueEndIndex -= 2; + } else if (argValue.endsWith("%")) { + factor = 0.01 * terminalEmulator.getCellHeightPixels() * terminalEmulator.getRows(); + intValueEndIndex -= 1; + } + try { + mHeight = (int) (factor * Integer.parseInt(argValue.substring(0, intValueEndIndex))); + } catch (Exception e) { + } + } else { + // `name` and `size` keys are not supported. + } + + if (lastParam) { + break; + } + } + + setState(ImageState.ARGUMENTS_READ); + + return index; + } + + + public synchronized boolean readImage(StringBuilder oscArgs, int index) { + if (!mIsMultipart) { + if (!ensureState(ImageState.ARGUMENTS_READ, "ImageState.readImage()")) { + return false; + } + + if (index < oscArgs.length()) { + int colonIndex = oscArgs.indexOf(":", index); + if (colonIndex >= 0 && colonIndex + 1 < oscArgs.length()) { + setState(ImageState.IMAGE_READING); + int imageStartIndex = colonIndex + 1; + + try { + // Appending can cause an increase in capacity and cause an OOM. + mEncodedImage.append(oscArgs.substring(imageStartIndex)); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + setStateFailed("Collecting singlepart image" + " in osc argument string failed: " + t.getMessage()); + return false; + } + + setState(ImageState.IMAGE_READ); + return true; + } + } + + setStateFailed("Failed to read singlepart image from index " + index + " in osc argument string: " + oscArgs); + return false; + } else { + if (mCurrentState != ImageState.IMAGE_READING && + !ensureState(ImageState.ARGUMENTS_READ, "ImageState.readImage()")) { + return false; + } + + // An empty `FilePart=` command could be received as well, so change state before `if` below. + setState(ImageState.IMAGE_READING); + + if (index < oscArgs.length()) { + try { + // Appending can cause an increase in capacity and cause an OOM. + mEncodedImage.append(oscArgs.substring(index)); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + setStateFailed("Collecting multipart image" + " in osc argument string failed: " + t.getMessage()); + return false; + } + return true; + } + + setStateFailed("Failed to read multipart image" + " in osc argument string: " + oscArgs); + return false; + } + } + + public synchronized boolean setMultiPartImageRead() { + if (!mIsMultipart) { + Logger.logError(mClient, LOG_TAG, "Attempting to call 'ImageState.setMultiPartImageRead()' for a singlepart image"); + return false; + } + + // A `FileEnd` command may have been received without a `FilePart=` command preceding it. + if (!ensureState(ImageState.IMAGE_READING, "ImageState.setMultiPartImageRead()")) { + return false; + } + + setState(ImageState.IMAGE_READ); + return true; + } + + + public synchronized boolean decodeImage() { + if (!ensureState(ImageState.IMAGE_READ, "ImageState.decodeImage()")) { + return false; + } + + String encodedImageString = null; + try { + if (mEncodedImage.length() < 1) { + setStateFailed("Cannot decoded an empty image"); + return false; + } + + while (mEncodedImage.length() % 4 != 0) { + mEncodedImage.append('='); + } + + encodedImageString = mEncodedImage.toString(); + + // Clear original encoded image from memory as it is no longer needed. + mEncodedImage.setLength(0); + mEncodedImage.trimToSize(); + + mDecodedImage = Base64.decode(encodedImageString, Base64.DEFAULT); + if (mDecodedImage == null || mDecodedImage.length < 2) { + setStateFailed("The decoded image is not valid: " + Arrays.toString(mDecodedImage) + "\nimage: " + encodedImageString); + return false; + } + + setState(ImageState.IMAGE_DECODED); + return true; + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) { + Logger.logError(mClient, LOG_TAG, "Failed to decode image: " + t.getMessage()); + System.gc(); + } else { + Logger.logStackTraceWithMessage(mClient, LOG_TAG, "Failed to decode image: " + encodedImageString, t); + } + setStateFailed(null); + return false; + } + } + +} diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java index 9758705bdc..38eccad1f8 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java @@ -1,187 +1,353 @@ package com.termux.terminal; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - import android.graphics.Bitmap; import android.graphics.BitmapFactory; -import android.graphics.Rect; - -import android.os.SystemClock; /** - * A circular buffer of {@link TerminalRow}:s which keeps notes about what is visible on a logical screen and the scroll - * history. - *

    - * See {@link #externalToInternalRow(int)} for how to map from logical screen rows to array indices. + * A terminal bitmap for images. */ public class TerminalBitmap { - public Bitmap bitmap; - public int cellWidth; - public int cellHeight; - public int scrollLines; - public int[] cursorDelta; - private static final String LOG_TAG = "TerminalBitmap"; - - - public TerminalBitmap(int num, WorkingTerminalBitmap sixel, int Y, int X, int cellW, int cellH, TerminalBuffer screen) { - Bitmap bm = sixel.bitmap; - bm = resizeBitmapConstraints(bm, sixel.width, sixel.height, cellW, cellH, screen.mColumns - X); - addBitmap(num, bm, Y, X, cellW, cellH, screen); + + public static final String LOG_TAG = "TerminalBitmap"; + + + protected final TerminalSessionClient mClient; + + protected int mBitmapNum; + protected Bitmap mBitmap; + + protected int mCellWidth; + protected int mCellHeight; + + protected int mScrollLines; + + protected int[] mCursorDelta; + + + protected TerminalBitmap(TerminalSessionClient client, int bitmapNum, Bitmap bitmap, + int cellWidth, int cellHeight, + int scrollLines, int[] cursorDelta) { + mClient = client; + + mBitmapNum = bitmapNum; + mBitmap = bitmap; + + mCellWidth = cellWidth; + mCellHeight = cellHeight; + + mScrollLines = scrollLines; + mCursorDelta = cursorDelta; + } + + + + /** Build a {@link TerminalBitmap} from a {@link TerminalSixel}. */ + public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, TerminalSixel terminalSixel, + int x, int y, int cellWidth, int cellHeight) { + try { + Bitmap bitmap = terminalSixel.getBitmap(); + bitmap = resizeBitmapConstrained(LOG_TAG, "sixel", terminalBuffer.getClient(), bitmap, + terminalSixel.getWidth(), terminalSixel.getHeight(), cellWidth, cellHeight, + terminalBuffer.mColumns - x); + if (bitmap == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from terminal sixel failed"); + return null; + } + + return buildOrThrow(terminalBuffer, bitmapNum, bitmap, + x, y, cellWidth, cellHeight); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from terminal sixel failed: " + t.getMessage()); + return null; + } } - public TerminalBitmap(int num, byte[] image, int Y, int X, int cellW, int cellH, int width, int height, boolean aspect, TerminalBuffer screen) { - Bitmap bm = null; - int imageHeight; - int imageWidth; - int newWidth = width; - int newHeight = height; - if (height > 0 || width > 0) { - BitmapFactory.Options options = new BitmapFactory.Options(); - options.inJustDecodeBounds = true; - try { - BitmapFactory.decodeByteArray(image, 0, image.length, options); - } catch (Exception e) { - Logger.logWarn(null, LOG_TAG, "Cannot decode image"); + + /** Build a {@link TerminalBitmap} from an image `byte[]`. */ + public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, byte[] image, + int x, int y, int cellWidth, int cellHeight, + int width, int height, boolean shouldPreserveAspectRatio) { + try { + Bitmap newBitmap; + int imageHeight; + int imageWidth; + int newWidth = width; + int newHeight = height; + + if (image == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Image data not set"); + return null; } - imageHeight = options.outHeight; - imageWidth = options.outWidth; - if (aspect) { - double wFactor = 9999.0; - double hFactor = 9999.0; - if (width > 0) { - wFactor = (double)width / imageWidth; + + if (height > 0 || width > 0) { + // Get image dimensions without creating a bitmap. + BitmapFactory.Options options = new BitmapFactory.Options(); + options.inJustDecodeBounds = true; + try { + BitmapFactory.decodeByteArray(image, 0, image.length, options); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logWarn(terminalBuffer.getClient(), LOG_TAG, + "Decode bitmap failed while creating" + + " terminal bitmap " + bitmapNum + " from image byte array: " + t.getMessage()); } - if (height > 0) { - hFactor = (double)height / imageHeight; + + + imageHeight = options.outHeight; + imageWidth = options.outWidth; + if (shouldPreserveAspectRatio) { + double wFactor = 9999.0; + double hFactor = 9999.0; + if (width > 0) { + wFactor = (double) width / imageWidth; + } + if (height > 0) { + hFactor = (double) height / imageHeight; + } + double factor = Math.min(wFactor, hFactor); + newWidth = (int) (factor * imageWidth); + newHeight = (int) (factor * imageHeight); + } else { + if (height <= 0) { + newHeight = imageHeight; + } + if (width <= 0) { + newWidth = imageWidth; + } } - double factor = Math.min(wFactor, hFactor); - newWidth = (int)(factor * imageWidth); - newHeight = (int)(factor * imageHeight); - } else { - if (height <= 0) { - newHeight = imageHeight; + + int scaleFactor = 1; + while (imageHeight >= 2 * newHeight * scaleFactor && imageWidth >= 2 * newWidth * scaleFactor) { + scaleFactor = scaleFactor * 2; } - if (width <= 0) { - newWidth = imageWidth; + + + // Create bitmap from image. + try { + if (scaleFactor > 1) { + // Subsample the original image to get a smaller image to save memory. + BitmapFactory.Options scaleOptions = new BitmapFactory.Options(); + scaleOptions.inSampleSize = scaleFactor; + newBitmap = BitmapFactory.decodeByteArray(image, 0, image.length, scaleOptions); + } else { + newBitmap = BitmapFactory.decodeByteArray(image, 0, image.length); + } + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Decode scaled bitmap for scale factor " + scaleFactor + " failed: " + t.getMessage()); + return null; } - } - int scaleFactor = 1; - while (imageHeight >= 2 * newHeight * scaleFactor && imageWidth >= 2 * newWidth * scaleFactor) { - scaleFactor = scaleFactor * 2; - } - BitmapFactory.Options scaleOptions = new BitmapFactory.Options(); - scaleOptions.inSampleSize = scaleFactor; - try { - bm = BitmapFactory.decodeByteArray(image, 0, image.length, scaleOptions); - } catch (Exception e) { - Logger.logWarn(null, LOG_TAG, "Out of memory, cannot decode image"); - bitmap = null; - return; - } - if (bm == null) { - Logger.logWarn(null, LOG_TAG, "Could not decode image"); - bitmap = null; - return; - } - int maxWidth = (screen.mColumns - X) * cellW; - if (newWidth > maxWidth) { - int cropWidth = bm.getWidth() * maxWidth / newWidth; + if (newBitmap == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Decoded scaled bitmap not set for scale factor " + scaleFactor); + return null; + } + + + // Crop the bitmap if it exceeds terminal bounds. + int maxWidth = (terminalBuffer.mColumns - x) * cellWidth; + if (newWidth > maxWidth) { + int cropWidth = newBitmap.getWidth() * maxWidth / newWidth; + try { + newBitmap = Bitmap.createBitmap(newBitmap, 0, 0, cropWidth, newBitmap.getHeight()); + newWidth = maxWidth; + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) { + // This is just a memory optimization. If it fails, + // continue (and probably fail later). + System.gc(); + } + + } + } + + + // Create final scaled bitmap. + try { + newBitmap = Bitmap.createScaledBitmap(newBitmap, newWidth, newHeight, true); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Create scaled bitmap failed: " + t.getMessage()); + return null; + } + } else { + // Create bitmap from image. try { - bm = Bitmap.createBitmap(bm, 0, 0, cropWidth, bm.getHeight()); - newWidth = maxWidth; - } catch(OutOfMemoryError e) { - // This is just a memory optimization. If it fails, - // continue (and probably fail later). + newBitmap = BitmapFactory.decodeByteArray(image, 0, image.length); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Create full bitmap failed: " + t.getMessage()); + return null; } } - try { - bm = Bitmap.createScaledBitmap(bm, newWidth, newHeight, true); - } catch(OutOfMemoryError e) { - Logger.logWarn(null, LOG_TAG, "Out of memory, cannot rescale image"); - bm = null; + + if (newBitmap == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed: New bitmap not set"); + return null; } - } else { - try { - bm = BitmapFactory.decodeByteArray(image, 0, image.length); - } catch (OutOfMemoryError e) { - Logger.logWarn(null, LOG_TAG, "Out of memory, cannot decode image"); + + + newBitmap = resizeBitmapConstrained(LOG_TAG, "image byte array", terminalBuffer.getClient(), newBitmap, + newBitmap.getWidth(), newBitmap.getHeight(), cellWidth, cellHeight, + terminalBuffer.mColumns - x); + TerminalBitmap terminalBitmap = build(terminalBuffer, bitmapNum, newBitmap, x, y, cellWidth, cellHeight); + if (terminalBitmap == null) { + return terminalBitmap; } - } - if (bm == null) { - Logger.logWarn(null, LOG_TAG, "Cannot decode image"); - bitmap = null; - return; + terminalBitmap.setCursorDelta(new int[] { + terminalBitmap.getScrollLines(), + (terminalBitmap.getBitmap().getWidth() + cellWidth - 1) / cellWidth}); + + return terminalBitmap; + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed: " + t.getMessage()); + return null; } + } - bm = resizeBitmapConstraints(bm, bm.getWidth(), bm.getHeight(), cellW, cellH, screen.mColumns - X); - addBitmap(num, bm, Y, X, cellW, cellH, screen); - cursorDelta = new int[] {scrollLines, (bitmap.getWidth() + cellW - 1) / cellW}; + + /** Build a {@link TerminalBitmap} from a {@link Bitmap}. */ + public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, Bitmap bitmap, + int x, int y, int cellWidth, int cellHeight) { + try { + return buildOrThrow(terminalBuffer, bitmapNum, bitmap, x, y, cellWidth, cellHeight); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from bitmap failed: " + t.getMessage()); + return null; + } } - private void addBitmap(int num, Bitmap bm, int Y, int X, int cellW, int cellH, TerminalBuffer screen) { - if (bm == null) { - bitmap = null; - return; + /** Build a {@link TerminalBitmap} from a {@link Bitmap}. */ + public static TerminalBitmap buildOrThrow(TerminalBuffer terminalBuffer, int bitmapNum, Bitmap bitmap, + int x, int y, int cellWidth, int cellHeight) throws Throwable { + if (bitmap == null) { + throw new IllegalArgumentException("Cannot create terminal bitmap from an unset bitmap"); } - int width = bm.getWidth(); - int height = bm.getHeight(); - cellWidth = cellW; - cellHeight = cellH; - int w = Math.min(screen.mColumns - X, (width + cellW - 1) / cellW); - int h = (height + cellH - 1) / cellH; + + int bitmapWidth = bitmap.getWidth(); + int bitmapHeight = bitmap.getHeight(); + int width = Math.min(terminalBuffer.mColumns - x, (bitmapWidth + cellWidth - 1) / cellWidth); + int height = (bitmapHeight + cellHeight - 1) / cellHeight; int s = 0; - for (int i=0; i cellW * Columns || (w % cellW) != 0 || (h % cellH) != 0) { - int newW = Math.min(cellW * Columns, ((w - 1) / cellW) * cellW + cellW); - int newH = ((h - 1) / cellH) * cellH + cellH; - try { - bm = resizeBitmap(bm, newW, newH); - } catch(OutOfMemoryError e) { - // Only a minor display glitch in this case - } + public static Bitmap resizeBitmapConstrained(String logTag, String label, TerminalSessionClient client, Bitmap bitmap, + int bitmapWidth, int bitmapHeight, + int cellWidth, int cellHeight, int columns) { + // Width and height must be multiples of the cell width and height. + // Bitmap should not extend beyond screen width. + Bitmap originalBitmap = bitmap; + if (bitmapWidth > cellWidth * columns || (bitmapWidth % cellWidth) != 0 || (bitmapHeight % cellHeight) != 0) { + int newBitmapWidth = Math.min(cellWidth * columns, ((bitmapWidth - 1) / cellWidth) * cellWidth + cellWidth); + int newBitmapHeight = ((bitmapHeight - 1) / cellHeight) * cellHeight + cellHeight; + bitmap = resizeBitmap(logTag, label, client, originalBitmap, newBitmapWidth, newBitmapHeight); + // Only a minor display glitch if resize failed. + return bitmap != null ? bitmap : originalBitmap; + } else { + return originalBitmap; } - return bm; } + } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java index 8e8620d393..fea81faf41 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java @@ -6,7 +6,6 @@ import java.util.HashMap; import android.graphics.Bitmap; -import android.graphics.BitmapFactory; import android.graphics.Rect; import android.os.SystemClock; @@ -19,6 +18,12 @@ */ public final class TerminalBuffer { + public static final String LOG_TAG = "TerminalBuffer"; + + + + private TerminalSessionClient mClient; + TerminalRow[] mLines; /** The length of {@link #mLines}. */ int mTotalRows; @@ -29,32 +34,74 @@ public final class TerminalBuffer { /** The index in the circular buffer where the visible screen starts. */ private int mScreenFirstRow = 0; - public HashMap bitmaps; - public WorkingTerminalBitmap workingBitmap; - private boolean hasBitmaps; - private long bitmapLastGC; + /** + * The {@link TerminalSixel} if a sixel command is being processed, from which the final + * {@link TerminalBitmap} is created. + */ + private TerminalSixel mTerminalSixel; + + + /** The map for bitmap number to the {@link TerminalBitmap} loaded in the terminal. */ + private final HashMap mTerminalBitmaps; + + /** The time since last garbage collection for all the {@link TerminalBitmap} that are loaded in the terminal. */ + private long mTerminalBitmapsLastGC; + + /** + * The bitmap number start for {@link #mTerminalBitmaps} keys. + * + * The bitmap number and coordinates are encoded in the `long` {@link TerminalRow#mStyle} for + * the `TerminalRow` character of a column by + * {@link TerminalBitmap#buildOrThrow(TerminalBuffer, int, Bitmap, int, int, int, int)} by + * getting encoded value from {@link TextStyle#encodeTerminalBitmap(int, int, int)}. + * The `TerminalRenderer.render()` then checks during rendering terminal output whether a + * character at a row/coloumn index is a bitmap instead of text by calling + * `TextStyle.isTerminalBitmap()`. + */ + public static final int TERMINAL_BITMAP__NUM_START = 0; + + /** + * The bitmap number end for {@link #mTerminalBitmaps} keys. + */ + public static final int TERMINAL_BITMAP__NUM_END = Integer.MAX_VALUE; + + + + + public TerminalBuffer(int columns, int totalRows, int screenRows) { + this(null, columns, totalRows, screenRows); + } /** * Create a transcript screen. * + * @param client the {@link TerminalSessionClient}. * @param columns the width of the screen in characters. * @param totalRows the height of the entire text area, in rows of text. * @param screenRows the height of just the screen, not including the transcript that holds lines that have scrolled off * the top of the screen. */ - public TerminalBuffer(int columns, int totalRows, int screenRows) { + public TerminalBuffer(TerminalSessionClient client, int columns, int totalRows, int screenRows) { + mClient = client; + mColumns = columns; mTotalRows = totalRows; mScreenRows = screenRows; mLines = new TerminalRow[totalRows]; blockSet(0, 0, columns, screenRows, ' ', TextStyle.NORMAL); - hasBitmaps = false; - bitmaps = new HashMap(); - bitmapLastGC = SystemClock.uptimeMillis(); + mTerminalBitmaps = new HashMap<>(); + mTerminalBitmapsLastGC = SystemClock.uptimeMillis(); + } + + + + public TerminalSessionClient getClient() { + return mClient; } + public String getTranscriptText() { return getSelectedText(0, -getActiveTranscriptRows(), mColumns, mScreenRows).trim(); } @@ -419,27 +466,9 @@ public void scrollDownOneLine(int topMargin, int bottomMargin, long style) { if (mLines[blankRow] == null) { mLines[blankRow] = new TerminalRow(mColumns, style); } else { - // find if a bitmap is completely scrolled out - Set used = new HashSet(); - if(mLines[blankRow].mHasBitmap) { - for (int column = 0; column < mColumns; column++) { - final long st = mLines[blankRow].getStyle(column); - if (TextStyle.isBitmap(st)) { - used.add((int)(st >> 16) & 0xffff); - } - } - TerminalRow nextLine = mLines[(blankRow + 1) % mTotalRows]; - if(nextLine.mHasBitmap) { - for (int column = 0; column < mColumns; column++) { - final long st = nextLine.getStyle(column); - if (TextStyle.isBitmap(st)) { - used.remove((int)(st >> 16) & 0xffff); - } - } - } - for(Integer bm: used) { - bitmaps.remove(bm); - } + // Remove bitmaps that are completely scrolled out. + if(mLines[blankRow].mHasTerminalBitmap) { + removeScrolledOutTerminalBitmaps(blankRow); } mLines[blankRow].clear(style); } @@ -528,7 +557,7 @@ public void setOrClearEffect(int bits, boolean setOrClear, boolean reverse, bool } } - public void clearTranscript() { + public synchronized void clearTranscript() { if (mScreenFirstRow < mActiveTranscriptRows) { Arrays.fill(mLines, mTotalRows + mScreenFirstRow - mActiveTranscriptRows, mTotalRows, null); Arrays.fill(mLines, 0, mScreenFirstRow, null); @@ -536,92 +565,197 @@ public void clearTranscript() { Arrays.fill(mLines, mScreenFirstRow - mActiveTranscriptRows, mScreenFirstRow, null); } mActiveTranscriptRows = 0; - bitmaps.clear(); - hasBitmaps = false; + clearTerminalBitmaps(); } - public Bitmap getSixelBitmap(int codePoint, long style) { - return bitmaps.get(TextStyle.bitmapNum(style)).bitmap; + + + public synchronized TerminalBitmap getTerminalBitmap(long style) { + int bitmapNum = TextStyle.getTerminalBitmapNum(style); + return bitmapNum >= TERMINAL_BITMAP__NUM_START ? mTerminalBitmaps.get(bitmapNum): null; } - public Rect getSixelRect(int codePoint, long style ) { - TerminalBitmap bm = bitmaps.get(TextStyle.bitmapNum(style)); - int x = TextStyle.bitmapX(style); - int y = TextStyle.bitmapY(style); - Rect r = new Rect(x * bm.cellWidth, y * bm.cellHeight, (x+1) * bm.cellWidth, (y+1) * bm.cellHeight); - return r; + public synchronized void clearTerminalBitmaps() { + mTerminalBitmaps.clear(); } - public void sixelStart(int width, int height) { - workingBitmap = new WorkingTerminalBitmap(width, height); + public synchronized Bitmap getSixelBitmap(long style) { + TerminalBitmap terminalBitmap = getTerminalBitmap(style); + return terminalBitmap != null ? terminalBitmap.mBitmap : null; } - public void sixelChar(int c, int rep) { - workingBitmap.sixelChar(c, rep); + + public synchronized Rect getSixelRect(long style) { + TerminalBitmap terminalBitmap = getTerminalBitmap(style); + if (terminalBitmap == null) { + return null; + } + + int x = TextStyle.getTerminalBitmapX(style); + int y = TextStyle.getTerminalBitmapY(style); + return new Rect( + x * terminalBitmap.mCellWidth, + y * terminalBitmap.mCellHeight, + (x + 1) * terminalBitmap.mCellWidth, + (y + 1) * terminalBitmap.mCellHeight); } - public void sixelSetColor(int col) { - workingBitmap.sixelSetColor(col); + + public synchronized void sixelStart(int width, int height) { + mTerminalSixel = TerminalSixel.build(getClient(), width, height); } - public void sixelSetColor(int col, int r, int g, int b) { - workingBitmap.sixelSetColor(col, r, g, b); + public synchronized int sixelEnd(int x, int y, int cellW, int cellH) { + if (mTerminalSixel == null) return 0; + + int bitmapNum = getFreeTerminalBitmapNum(); + if (bitmapNum < TERMINAL_BITMAP__NUM_START) { + Logger.logError(mClient, LOG_TAG, "Cannot create more than " + TERMINAL_BITMAP__NUM_END + " bitmaps"); + return 0; + } + + TerminalBitmap terminalBitmap = TerminalBitmap.build(this, bitmapNum, mTerminalSixel, x, y, cellW, cellH); + mTerminalSixel = null; + if (terminalBitmap == null || terminalBitmap.getBitmap() == null) { + return 0; + } + mTerminalBitmaps.put(bitmapNum, terminalBitmap); + + doTerminalBitmapsGC(30000); + return terminalBitmap.mScrollLines; } - private int findFreeBitmap() { - int i = 0; - while (bitmaps.containsKey(i)) { - i++; + /** Clears the {@link #mTerminalSixel} by setting it to `null`. */ + public synchronized void sixelClear() { + mTerminalSixel = null; + } + + public synchronized boolean sixelReadData(int codePoint, int repeat) { + // If an error occurred during processing (like OOM), then remaining sixel command is + // completely read, but is ignored. + if (mTerminalSixel != null) { + if (!mTerminalSixel.readData(codePoint, repeat)) { + // Ignore further commands/data. + mTerminalSixel = null; + return false; + } } - return i; + return true; } - public int sixelEnd(int Y, int X, int cellW, int cellH) { - int num = findFreeBitmap(); - bitmaps.put(num, new TerminalBitmap(num, workingBitmap, Y, X, cellW, cellH, this)); - workingBitmap = null; - if (bitmaps.get(num).bitmap == null) { - bitmaps.remove(num); - return 0; + public synchronized boolean sixelResize(int sixelWidth, int sixelHeight) { + // If an error occurred during processing (like OOM), then remaining sixel command is + // completely read, but is ignored. + if (mTerminalSixel != null) { + if (!mTerminalSixel.resize(sixelWidth, sixelHeight)) { + // Ignore further commands/data. + mTerminalSixel = null; + return false; + } } - hasBitmaps = true; - bitmapGC(30000); - return bitmaps.get(num).scrollLines; + return true; + } + + public synchronized void sixelSetColor(int color) { + if (mTerminalSixel != null) + mTerminalSixel.setColor(color); + } + + public synchronized void sixelSetRGBColor(int color, int r, int g, int b) { + if (mTerminalSixel != null) + mTerminalSixel.setRGBColor(color, r, g, b); } - public int[] addImage(byte[] image, int Y, int X, int cellW, int cellH, int width, int height, boolean aspect) { - int num = findFreeBitmap(); - bitmaps.put(num, new TerminalBitmap(num, image, Y, X, cellW, cellH, width, height, aspect, this)); - if (bitmaps.get(num).bitmap == null) { - bitmaps.remove(num); - return new int[] {0,0}; + + + private synchronized int getFreeTerminalBitmapNum() { + int bitmapNum = TERMINAL_BITMAP__NUM_START; + while (mTerminalBitmaps.containsKey(bitmapNum)) { + bitmapNum++; + if (bitmapNum == TERMINAL_BITMAP__NUM_END) { + return -1; + } } - hasBitmaps = true; - bitmapGC(30000); - return bitmaps.get(num).cursorDelta; + return bitmapNum; } - public void bitmapGC(int timeDelta) { - if (!hasBitmaps || bitmapLastGC + timeDelta > SystemClock.uptimeMillis()) { + + public synchronized int[] addTerminalBitmapForImage(byte[] image, int x, int y, int cellW, int cellH, int width, int height, boolean shouldPreserveAspectRatio) { + int bitmapNum = getFreeTerminalBitmapNum(); + if (bitmapNum < TERMINAL_BITMAP__NUM_START) { + Logger.logError(mClient, LOG_TAG, "Cannot create more than " + TERMINAL_BITMAP__NUM_END + " bitmaps"); + return new int[] {0, 0}; + } + + TerminalBitmap terminalBitmap = TerminalBitmap.build(this, bitmapNum, image, x, y, + cellW, cellH, width, height, shouldPreserveAspectRatio); + if (terminalBitmap == null || terminalBitmap.getBitmap() == null) { + return new int[] {0, 0}; + } + mTerminalBitmaps.put(bitmapNum, terminalBitmap); + + doTerminalBitmapsGC(30000); + return terminalBitmap.mCursorDelta; + } + + + /** Remove bitmaps that are completely scrolled out. */ + public synchronized void removeScrolledOutTerminalBitmaps(int row) { + Set bitmapsToRemove = new HashSet<>(); + + for (int column = 0; column < mColumns; column++) { + long columnStyle = mLines[row].getStyle(column); + int bitmapNum = TextStyle.getTerminalBitmapNum(columnStyle); + if (bitmapNum >= TERMINAL_BITMAP__NUM_START) { + bitmapsToRemove.add(bitmapNum); + } + } + + if (row + 1 < mTotalRows) { + TerminalRow nextLine = mLines[row + 1]; + if (nextLine.mHasTerminalBitmap) { + for (int column = 0; column < mColumns; column++) { + long columnStyle = nextLine.getStyle(column); + int bitmapNum = TextStyle.getTerminalBitmapNum(columnStyle); + if (bitmapNum >= TERMINAL_BITMAP__NUM_START) { + bitmapsToRemove.add(bitmapNum); + } + } + } + } + + for(Integer bitmapStyle : bitmapsToRemove) { + mTerminalBitmaps.remove(bitmapStyle); + } + } + + public synchronized void doTerminalBitmapsGC(int timeDelta) { + if (mTerminalBitmaps.isEmpty() || mTerminalBitmapsLastGC + timeDelta > SystemClock.uptimeMillis()) { return; } - Set used = new HashSet(); + + Set bitmapsToKeep = new HashSet<>(); + for (int line = 0; line < mLines.length; line++) { - if(mLines[line] != null && mLines[line].mHasBitmap) { + if(mLines[line] != null && mLines[line].mHasTerminalBitmap) { for (int column = 0; column < mColumns; column++) { - final long st = mLines[line].getStyle(column); - if (TextStyle.isBitmap(st)) { - used.add((int)(st >> 16) & 0xffff); + long style = mLines[line].getStyle(column); + int bitmapNum = TextStyle.getTerminalBitmapNum(style); + if (bitmapNum >= TERMINAL_BITMAP__NUM_START) { + bitmapsToKeep.add(bitmapNum); } } } } - Set keys = new HashSet(bitmaps.keySet()); - for (Integer bn: keys) { - if (!used.contains(bn)) { - bitmaps.remove(bn); + + Set bitmapNums = new HashSet<>(mTerminalBitmaps.keySet()); + for (Integer bitmapNum: bitmapNums) { + if (!bitmapsToKeep.contains(bitmapNum)) { + mTerminalBitmaps.remove(bitmapNum); } } - bitmapLastGC = SystemClock.uptimeMillis(); + + mTerminalBitmapsLastGC = SystemClock.uptimeMillis(); } + } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index 1fe43023bc..64473e019f 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -3,7 +3,6 @@ import android.util.Base64; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; import java.util.Objects; @@ -321,6 +320,33 @@ public final class TerminalEmulator { private boolean ESC_DCS__ESC = false; + /** Whether processing a sixel `DCS q s..s ST` or `DCS P1; P2; P3; q s..s ST` command to create a {@link TerminalSixel}. */ + private boolean ESC_DCS__SIXEL = false; + + /** Whether to check if sixel command is being received when processing a DCS command. */ + private boolean ESC_DCS__CHECK_IF_SIXEL = true; + + /** The command part number in case a long sixel command was broken into parts for processing. */ + private int mSixelCommandPartNum; + + /** + * The capacity to set for {@link #mTerminalControlArgs} used to store sixel commands before processing. + * + * See also {@link #ensureTerminalControlArgsCapacity(int)}. + */ + private Integer mSixelArgsCapacity; + + /** + * The initial capacity for sixel args stored in {@link #mTerminalControlArgs}. + */ + private static final int SIXEL_ARGS__INITIAL_CAPACITY = 256; + + + + /** The {@link ITermImage} if an iTerm image command is being processed. */ + private ITermImage mITermImage; + + /** * True if the current escape sequence should continue, false if the current escape sequence should be terminated. @@ -331,12 +357,6 @@ public final class TerminalEmulator { /** The current state of the escape sequence state machine. One of the ESC_* constants. */ private int mEscapeState; - private boolean ESC_DCS_escape = false; - private boolean ESC_DCS_sixel = false; - private ArrayList ESC_OSC_data; - private int ESC_OSC_colon = 0; - private boolean ESC_OSC_outofmem = false; - private final SavedScreenState mSavedStateMain = new SavedScreenState(); private final SavedScreenState mSavedStateAlt = new SavedScreenState(); @@ -413,13 +433,6 @@ public final class TerminalEmulator { private static final String LOG_TAG = "TerminalEmulator"; - private int cellW = 12, cellH = 12; - - public void setCellSize(int w, int h) { - cellW = w; - cellH = h; - } - private boolean isDecsetInternalBitSet(int bit) { return (mCurrentDecSetFlags & bit) != 0; } @@ -474,8 +487,8 @@ static int mapDecSetBitToInternalBit(int decsetBit) { public TerminalEmulator(TerminalOutput session, int columns, int rows, int cellWidthPixels, int cellHeightPixels, Integer transcriptRows, TerminalSessionClient client) { mSession = session; - mScreen = mMainBuffer = new TerminalBuffer(columns, getTerminalTranscriptRows(transcriptRows), rows); - mAltBuffer = new TerminalBuffer(columns, rows, rows); + mScreen = mMainBuffer = new TerminalBuffer(client, columns, getTerminalTranscriptRows(transcriptRows), rows); + mAltBuffer = new TerminalBuffer(client, columns, rows, rows); mClient = client; mRows = rows; mColumns = columns; @@ -491,10 +504,28 @@ public void updateTerminalSessionClient(TerminalSessionClient client) { setCursorBlinkState(true); } + + public TerminalBuffer getScreen() { return mScreen; } + public int getRows() { + return mRows; + } + + public int getColumns() { + return mColumns; + } + + public int getCellWidthPixels() { + return mCellWidthPixels; + } + + public int getCellHeightPixels() { + return mCellHeightPixels; + } + public boolean isAlternateBufferActive() { return mScreen == mAltBuffer; } @@ -717,6 +748,8 @@ private void processByte(byte byteToProcess) { } public void processCodePoint(int b) { + mScreen.doTerminalBitmapsGC(300000); + if (mEscapeState == ESC_OSC && mIsFastPathOsc) { mContinueSequence = false; receiveOsc(b); @@ -748,7 +781,6 @@ public void processCodePoint(int b) { return; } - mScreen.bitmapGC(300000); switch (b) { case 0: // Null character (NUL, ^@). Do nothing. break; @@ -783,14 +815,17 @@ public void processCodePoint(int b) { case 10: // Line feed (LF, \n). case 11: // Vertical tab (VT, \v). case 12: // Form feed (FF, \f). - if((mEscapeState != ESC_DCS || !ESC_DCS_sixel) && ESC_OSC_colon <= 0) { - // Ignore CR/LF inside sixels or iterm2 data + // Ignore CR/LF inside DCS by default (including sixel) or OSC if requested (like for iTerm). + if (! + (mEscapeState == ESC_DCS || + ((mEscapeState == ESC_OSC || mEscapeState == ESC_OSC__ESC) && mIgnoreCrLfForOsc))) { doLinefeed(); } break; case 13: // Carriage return (CR, \r). - if((mEscapeState != ESC_DCS || !ESC_DCS_sixel) && ESC_OSC_colon <= 0) { - // Ignore CR/LF inside sixels or iterm2 data + if (! + (mEscapeState == ESC_DCS || + ((mEscapeState == ESC_OSC || mEscapeState == ESC_OSC__ESC) && mIgnoreCrLfForOsc))) { setCursorCol(mLeftMargin); } break; @@ -1099,24 +1134,34 @@ public void processCodePoint(int b) { * Do {@link #ESC_DCS}. Check its docs for more info. */ private void doDcs(final int b) { - boolean firstSixel = false; - if (!ESC_DCS_sixel && (b=='$' || b=='-' || b=='#')) { - //Check if sixel sequence that needs breaking - String dcs = mTerminalControlArgs.toString(); - if (dcs.matches("[0-9;]*q.*")) { - firstSixel = true; - } - } - if ( // End of DCS if string terminator ST `ESC \` received. - (ESC_DCS__ESC && b == '\\') - // Sixel sequences may be very long. '$' and '!' are natural for breaking the sequence. - || firstSixel || (ESC_DCS_sixel && (b=='$' || b=='-' || b=='#')) + (ESC_DCS__ESC && b == '\\') || + // If sixel continuation after sixel start and a + // Color Introducer `#`, Graphics Repeat Introducer `!` or Raster Attributes `"` + // command is received, then process any previous commands, or if end of input + // with a ST received. + // If `b` is a Color Introducer `#`, Graphics Repeat Introducer `!` or Raster Attributes `"` + // command, then it is added to buffer in code below and more input is waited for as + // further arguments need to be received for its command before it can be processed, + // which is not until the next command is received. + // We wait till at least `TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2` commands string has + // been received. The divide by 2 is done since if near the max length, a new command + // starts and it does not end before the max length, then `Terminal control args overflow + // error would occur. + // If the first command has been fully received, then we run it immediately in case + // it is the Raster Attributes command containing the "rough" horizontal and vertical + // size of image, which is used to set the capacity of the `mTerminalControlArgs` buffer + // and also resize the bitmap, so that memory allocations are avoided if possible. + // `mTerminalControlArgs.length() > 1` is done so that loop does not engage on first + // character after `q` and only after first command has been fully received. + (ESC_DCS__SIXEL && ((b == '#' || b == '!' || b == '"') && + ((mTerminalControlArgs.length() >= (TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2)) || (mTerminalControlArgs.length() > 1 && mSixelCommandPartNum == 1)))) ) { String dcs = mTerminalControlArgs.toString(); - // DCS $ q P t ST. Request Status String (DECRQSS) + // Request Selection or Setting (DECRQSS) `DCS $ q P t ST`. + // - https://vt100.net/docs/vt510-rm/DECRQSS.html if (dcs.startsWith("$q")) { if (dcs.equals("$q\"p")) { // DECSCL, conformance level, http://www.vt100.net/docs/vt510-rm/DECSCL: @@ -1214,93 +1259,46 @@ private void doDcs(final int b) { Logger.logError(mClient, LOG_TAG, "Invalid device termcap/terminfo name of odd length: " + part); } } - } else if (ESC_DCS_sixel || dcs.matches("[0-9;]*q.*")) { - int pos = 0; - if (!ESC_DCS_sixel) { - ESC_DCS_sixel = true; - mScreen.sixelStart(100, 100); - while (dcs.codePointAt(pos) != 'q') { - pos++; - } - pos++; - } - if (b=='$' || b=='-') { - // Add to string - dcs = dcs + (char)b; - } - int rep = 1; - while (pos < dcs.length()) { - if (dcs.codePointAt(pos) == '"') { - pos++; - int args[]={0,0,0,0}; - int arg = 0; - while (pos < dcs.length() && ((dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') || dcs.codePointAt(pos) == ';')) { - if (dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { - args[arg] = args[arg] * 10 + dcs.codePointAt(pos) - '0'; - } else { - arg++; - if (arg > 3) { - break; - } - } - pos++; - } - if (pos == dcs.length()) { - break; - } - } else if (dcs.codePointAt(pos) == '#') { - int col = 0; - pos++; - while (pos < dcs.length() && dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { - col = col * 10 + dcs.codePointAt(pos++) - '0'; - } - if (pos == dcs.length() || dcs.codePointAt(pos) != ';') { - mScreen.sixelSetColor(col); - } else { - pos++; - int args[]={0,0,0,0}; - int arg = 0; - while (pos < dcs.length() && ((dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') || dcs.codePointAt(pos) == ';')) { - if (dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { - args[arg] = args[arg] * 10 + dcs.codePointAt(pos) - '0'; - } else { - arg++; - if (arg > 3) { - break; - } - } - pos++; - } - if (args[0] == 2) { - mScreen.sixelSetColor(col, args[1], args[2], args[3]); - } - } - } else if (dcs.codePointAt(pos) == '!') { - rep = 0; - pos++; - while (pos < dcs.length() && dcs.codePointAt(pos) >= '0' && dcs.codePointAt(pos) <= '9') { - rep = rep * 10 + dcs.codePointAt(pos++) - '0'; - } - } else if (dcs.codePointAt(pos) == '$' || dcs.codePointAt(pos) == '-' || (dcs.codePointAt(pos) >= '?' && dcs.codePointAt(pos) <= '~')) { - mScreen.sixelChar(dcs.codePointAt(pos++), rep); - rep = 1; - } else { - pos++; - } + } + // If `s..s` or `ST` received from Sixel Device Control String `DCS q s..s ST` or `DCS P1; P2; P3; q s..s ST` command. + else if (ESC_DCS__SIXEL) { + mSixelCommandPartNum++; + + boolean isValidDcs = processSixelDcs(dcs); + + if (!isValidDcs) { + clearTerminalControlArgs(); + clearDcsTypeVariables(); + finishSequence(); + return; } - if (b == '\\') { - ESC_DCS_sixel = false; - int n = mScreen.sixelEnd(mCursorRow, mCursorCol, cellW, cellH); - for(;n>0;n--) { + + if (ESC_DCS__ESC && b == '\\') { + int n = mScreen.sixelEnd(mCursorCol, mCursorRow, mCellWidthPixels, mCellHeightPixels); + for(; n > 0; n--) { doLinefeed(); } + + // Clear DCS args buffer and variables and finish sequence. } else { - mTerminalControlArgs.setLength(0); - if (b=='#') { - mTerminalControlArgs.appendCodePoint(b); + ESC_DCS__ESC = false; + + // Clear DCS args buffer to receive further new input in empty buffer. + clearTerminalControlArgs(); + + // Increase capacity to expected capacity if `Raster Attributes` command + // was sent with image width and height, or to default + // `SIXEL_ARGS__INITIAL_CAPACITY` set by `startIfSixelDcs()`. + if (mSixelArgsCapacity != null) { + ensureTerminalControlArgsCapacity(mSixelArgsCapacity); } - // Do not finish sequence - continueSequence(mEscapeState); + + // If `b` is a Color Introducer `#`, Graphics Repeat Introducer `!` or Raster Attributes `"` + // command, then add to buffer and wait for more input as further arguments + // need to be received for its command before it can be processed, which is + // not until the next command is received. + if (!collectTerminalControlArgs(b)) return; + return; } } else { @@ -1316,14 +1314,310 @@ private void doDcs(final int b) { ESC_DCS__ESC = false; if (!collectTerminalControlArgs(b)) return; + + if (ESC_DCS__CHECK_IF_SIXEL && !ESC_DCS__SIXEL) { + // Check if `DCS q` or `DCS P1; P2; P3; q` received from Sixel + // Device Control String `DCS q s..s ST` or `DCS P1; P2; P3; q s..s ST` command. + // If received, then wait for more input after `q`. + if (b == 'q') { + startIfSixelDcs(); + } else if (b == ';' || (b >= '0' && b <= '9')) { + // Ignore. + } else { + ESC_DCS__CHECK_IF_SIXEL = false; + } + } } } public void clearDcsTypeVariables() { ESC_DCS__ESC = false; mIsFastPathDcs = false; + + ESC_DCS__SIXEL = false; + ESC_DCS__CHECK_IF_SIXEL = true; + mSixelCommandPartNum = 0; + mSixelArgsCapacity = null; + mScreen.sixelClear(); + } + + + + private void startIfSixelDcs() { + int[] sixelDcsSetupArgs = getSixelDcsSetupArgs(mTerminalControlArgs.toString(), 0); + if (sixelDcsSetupArgs != null) { + mIsFastPathDcs = true; + ESC_DCS__SIXEL = true; + ESC_DCS__CHECK_IF_SIXEL = false; + mSixelCommandPartNum = 1; + + // Do not actually increase capacity yet, as it will be increased by `doDcs()` after + // first command has been received, which is checked to see if its a `Raster Attributes` + // command with image width and height to calculate expected capacity. + mSixelArgsCapacity = SIXEL_ARGS__INITIAL_CAPACITY; + + // The `P1; P2; P3;` arguements in `sixelDcsSetupArgs` are ignored as they are not supported currently (if ever). + mScreen.sixelStart(100, 100); + clearTerminalControlArgs(); + } + } + + private int[] getSixelDcsSetupArgs(String dcs, int index) { + int[] args = {/* `P1=0`/`2:1` */ 0, /* `P2=0` */ 0, /* `P3=0` */ 0}; + + if (dcs.charAt(index) == 'q') return args; + + char ch; + + int arg = 0; boolean incArg = false; + while (index < dcs.length()) { + ch = dcs.charAt(index); + if (ch >= '0' && ch <= '9') { + if (incArg) { arg++; incArg = false; } + args[arg] = args[arg] * 10 + ch - '0'; + if (args[arg] < 0) { // Overflow. + break; + } + index++; + } else if (ch == ';') { + index++; + + if (arg == 2) { + if (index < dcs.length()) { + if (dcs.charAt(index) == 'q') { + return args; + } else { + // Must be some other command, so no need to check again. + ESC_DCS__CHECK_IF_SIXEL = false; + } + } + break; + } + + incArg = true; + } else { + break; + } + } + + return null; + } + + private boolean processSixelDcs(String dcs) { + int index = 0; + + char ch; + int repeat = 1; + int color; + boolean isValidDcs = true; + + while (index < dcs.length()) { + ch = dcs.charAt(index); + + if ( + // Sixel data characters in the range of `?` (0x3F) to `~` (0x7E). + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.2.1 + (ch >= '?' && ch <= '~') + // Graphics Carriage Return `$`. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.4 + || ch == '$' + // Graphics New Line `-`. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.5 + || ch == '-' + ) { + mScreen.sixelReadData(ch, repeat); + index++; + repeat = 1; + } + // Color Introducer `#` + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.3 + else if (ch == '#') { + index++; // Consume '#'. + + color = 0; + while (index < dcs.length()) { + ch = dcs.charAt(index); + if (ch >= '0' && ch <= '9') { + color = color * 10 + ch - '0'; + if (color > 255) { + Logger.logError(mClient, LOG_TAG, "The sixel color command Pc value " + color + " is not between 0-255 at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + index++; + } else { + break; + } + } + + if (!isValidDcs) { + break; + } + + // Basic Colors `# Pc` + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.3.1 + if (index == dcs.length() || dcs.charAt(index) != ';') { + mScreen.sixelSetColor(color); + } + // HLS or RGB Colors `# Pc; Pu; Px; Py; Pz` + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.3.2 + else if (dcs.charAt(index) == ';') { + index++; // Consume ';'. + + int[] args = {0, 0, 0, 0}; + int arg = 0; boolean incArg = false; + while (index < dcs.length()) { + ch = dcs.charAt(index); + if (ch >= '0' && ch <= '9') { + if (incArg) { arg++; incArg = false; } + args[arg] = args[arg] * 10 + ch - '0'; + if (arg == 0) { // Pu must equal 1 or 2. + if ((args[arg] != 1 && args[arg] != 2)) { + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command Pu value " + args[arg] + " is not 1 or 2 at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } else { + int limit = 100; + if (args[0] == 1 && arg == 1) limit = 360; + if (args[arg] > limit) { + String argName = ""; + switch (arg) { case 1: argName = "pX"; break; case 2: argName = "pY"; break; case 3: argName = "pZ"; break; } + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command " + argName + " value " + args[arg] + " is not between 0-" + limit + " at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } + } else if (ch == ';') { + if (arg == 3) { // Pz must not end with a ';'. + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command Pz value " + args[3] + " must not end with a semicolon ';' at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + + incArg = true; + } else { + break; + } + index++; + } + + if (isValidDcs && arg == 3) { // If complete spec is received and is valid. + if (args[0] == 2) { // Only RGB is supported. + mScreen.sixelSetRGBColor(color, args[1], args[2], args[3]); + } + } else { + if (isValidDcs) + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command expected 4 arguments at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } + } + // Graphics Repeat Introducer `! Pn character`. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.1 + else if (ch == '!') { + index++; // Consume '!'. + + repeat = 0; + while (index < dcs.length()) { + ch = dcs.charAt(index); + if (ch >= '0' && ch <= '9') { + repeat = repeat * 10 + ch - '0'; + if (repeat > TerminalSixel.SIXEL__MAX_REPEAT) { + Logger.logError(mClient, LOG_TAG, "The sixel repeat command Pn value " + repeat + " is greater than max repeat value " + + TerminalSixel.SIXEL__MAX_REPEAT + " at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + index++; + } else { + break; + } + } + + if (!isValidDcs) { + break; + } + } + // Raster Attributes `" Pan; Pad; Ph; Pv` + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.2 + else if (ch == '"') { + index++; // Consume '"'. + + int[] args = {0, 0, 0, 0}; + int arg = 0; boolean incArg = false; + while (index < dcs.length()) { + ch = dcs.charAt(index); + if (ch >= '0' && ch <= '9') { + if (incArg) { arg++; incArg = false; } + args[arg] = args[arg] * 10 + ch - '0'; + if (args[arg] < 0) { // Overflow. + String argName = ""; + switch (arg) { case 0: argName = "Pan"; break; case 1: argName = "Pad"; break; case 2: argName = "pH"; break; case 3: argName = "pV"; break; } + Logger.logError(mClient, LOG_TAG, "The sixel raster command " + argName + " value overflow at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } else if (ch == ';') { + if (arg == 3) { // Pv must not end with a ';'. + Logger.logError(mClient, LOG_TAG, "The sixel raster command Pv value " + args[3] + " must not end with a semicolon ';' at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + incArg = true; + } else { + break; + } + index++; + } + + if (isValidDcs && arg == 3) { // If complete spec is received and is valid. + // Raster pixel aspect ratio is not supported currently. + // Raster "rough" horizontal and vertical size of image may be sent at start of + // sixel data string, like done by `img2sixel`, so increase sixel commands args + // buffer capacity (`mTerminalControlArgs`) and resize sixel bitmap in + // `TerminalSixel` at start, instead of having to keep resizing buffer/bitmap + // as more sixel data is received, which has a performance hit due to + // memory reallocations and copying. + int sixelWidth = args[2]; // `Ph` + int sixelHeight = args[3]; // `Pv` + if (sixelWidth > 0 && sixelHeight > 0) { + // 2% extra for sixel commands/parameters in addition to image data. + int sixelArgsExpectedLength = (int) (sixelWidth * sixelHeight * 1.02); + // If sixel commands are too long, they are divided into parts, and if a + // new command starts near `TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2`, it could + // contain image data for 1 pixel line of image width, so add that. + int sixelArgsPartsExpectedLength = (int) ((((double) TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2) + sixelWidth) * 1.02); + int sixelArgsExpectedCapacity = Math.min(sixelArgsPartsExpectedLength, sixelArgsExpectedLength); + if (sixelArgsExpectedCapacity > SIXEL_ARGS__INITIAL_CAPACITY) { + mSixelArgsCapacity = sixelArgsExpectedCapacity; + } + + mScreen.sixelResize(sixelWidth, sixelHeight); + } + } else { + if (isValidDcs) + Logger.logError(mClient, LOG_TAG, "The sixel raster command expected 4 arguments at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } + else if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r') { + index++; + } else { + // Invalid character. + Logger.logError(mClient, LOG_TAG, "Invalid character '" + ch + "' (" + (byte) ch + ") at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } + + return isValidDcs; } + + private int nextTabStop(int numTabs) { for (int i = mCursorCol + 1; i < mColumns; i++) if (mTabStop[i] && --numTabs == 0) return Math.min(i, mRightMargin); @@ -1749,7 +2043,6 @@ private void doEsc(int b) { clearTerminalControlArgs(); clearOscTypeVariables(); continueSequence(ESC_OSC); - ESC_OSC_colon = -1; break; case '>': // DECKPNM setDecsetinternalBit(DECSET_BIT_APPLICATION_KEYPAD, false); @@ -2312,29 +2605,6 @@ private void receiveOsc(final int b) { if (mOscType == -1) { setOscTypeVariables(); } - - if (ESC_OSC_colon == -1 && b == ':') { - // Collect base64 data for OSC 1337 - ESC_OSC_colon = mTerminalControlArgs.length(); - ESC_OSC_data = new ArrayList(65536); - ESC_OSC_outofmem = false; - } else if (ESC_OSC_colon >= 0 && mTerminalControlArgs.length() - ESC_OSC_colon == 4) { - if (!ESC_OSC_outofmem) { - try { - byte[] decoded = Base64.decode(mTerminalControlArgs.substring(ESC_OSC_colon), 0); - for (int i = 0 ; i < decoded.length; i++) { - ESC_OSC_data.add(decoded[i]); - } - } catch(Exception e) { - // Ignore non-Base64 data. - } catch(OutOfMemoryError e) { - // Out of memory - // Keep decoding, but fo not collect the data - ESC_OSC_outofmem = true; - } - } - mTerminalControlArgs.setLength(ESC_OSC_colon); - } break; } } @@ -2385,6 +2655,16 @@ void setOscTypeVariables() { if (mOscType >= 0) { Integer terminalControlArgsCapacity = null; switch (mOscType) { + case 1337: // iTerm image command sends the base64 encoded image, do not run complex logic for each byte. + mIsFastPathOsc = true; + mIgnoreCrLfForOsc = true; + // Expect large amount of data for image bytes. + // `imgcat` utility splits image bytes into 200-byte chunks when sending with `FilePart=` commands. + // - https://github.com/gnachman/iTerm2-shell-integration/blob/d1d4012068c3c6761d5676c28ed73e0e2df2b715/utilities/imgcat#L89 + // > Older versions of tmux have a limit of 256 bytes for the entire sequence. + // - https://iterm2.com/documentation-images.html + terminalControlArgsCapacity = 256; + break; } if (terminalControlArgsCapacity != null) { @@ -2411,8 +2691,6 @@ public void clearOscTypeVariables() { */ private void doOsc(String bellOrStringTerminator) { int value = -1; - int osc_colon = ESC_OSC_colon; - ESC_OSC_colon = -1; String textParameter = ""; int argsLength = mTerminalControlArgs.length(); @@ -2420,7 +2698,10 @@ private void doOsc(String bellOrStringTerminator) { for (int i = 0; i < argsLength; i++) { char b = mTerminalControlArgs.charAt(i); if (b == ';') { - textParameter = mTerminalControlArgs.substring(i + 1); + // Do not make a copy of `mTerminalControlArgs` for lengthy commands. + if (value != 1337) { + textParameter = mTerminalControlArgs.substring(i + 1); + } break; } else if (b >= '0' && b <= '9') { value = ((value < 0) ? 0 : value * 10) + (b - '0'); @@ -2550,109 +2831,126 @@ private void doOsc(String bellOrStringTerminator) { break; case 119: // Reset highlight color. break; - case 1337: // iTerm extemsions - if (textParameter.startsWith("File=")) { - int pos = 5; - boolean inline = false; - boolean aspect = true; - int width = -1; - int height = -1; - while (pos < textParameter.length()) { - int eqpos = textParameter.indexOf('=', pos); - if (eqpos == -1) { - break; + case 1337: // iTerm image + // - https://iterm2.com/documentation-images.html + // - https://iterm2.com/documentation-escape-codes.html + String controlCommandPrefix = mTerminalControlArgs.substring(5, Math.min(19, argsLength)); + + if (controlCommandPrefix.startsWith("File=") || + controlCommandPrefix.startsWith("MultipartFile=") || + controlCommandPrefix.startsWith("FilePart=") || + controlCommandPrefix.equals("FileEnd")) { + + ITermImage iTermImage = null; + boolean oscArgsCleared = false; + int index; + // `File = [optional arguments] : base-64 encoded file contents ^G` + if (controlCommandPrefix.startsWith("File=")) { + if (mITermImage != null) { + Logger.logWarn(mClient, LOG_TAG, "A new iTerm 'File' command received while already processing a 'MultipartFile' command"); + mITermImage = null; // Unset old image. } - int semicolonpos = textParameter.indexOf(';', eqpos); - if (semicolonpos == -1) { - semicolonpos = textParameter.length() - 1; - } - String k = textParameter.substring(pos, eqpos); - String v = textParameter.substring(eqpos + 1, semicolonpos); - pos = semicolonpos + 1; - if (k.equalsIgnoreCase("inline")) { - inline = v.equals("1"); - } - if (k.equalsIgnoreCase("preserveAspectRatio")) { - aspect = ! v.equals("0"); - } - if (k.equalsIgnoreCase("width")) { - double factor = cellW; - int div = 1; - int e = v.length(); - if (v.endsWith("px")) { - factor = 1; - e -= 2; - } else if (v.endsWith("%")) { - factor = 0.01 * cellW * mColumns; - e -= 1; - } - try { - width = (int)(factor * Integer.parseInt(v.substring(0,e))); - } catch(Exception ex) { - } - } - if (k.equalsIgnoreCase("height")) { - double factor = cellH; - int div = 1; - int e = v.length(); - if (v.endsWith("px")) { - factor = 1; - e -= 2; - } else if (v.endsWith("%")) { - factor = 0.01 * cellH * mRows; - e -= 1; - } - try { - height = (int)(factor * Integer.parseInt(v.substring(0,e))); - } catch(Exception ex) { + + iTermImage = new ITermImage(mClient, /* multiPart */ false); + if ((index = iTermImage.readArguments(this, mTerminalControlArgs, /* `1337;File=` */ 10)) < 10 || + !iTermImage.readImage(mTerminalControlArgs, index)) { + iTermImage = null; + } else { + // Free image data from memory held in osc command arguments as it is no longer needed. + clearTerminalControlArgs(); + oscArgsCleared = true; + if (!iTermImage.decodeImage()) { + iTermImage = null; } } } - if (!inline) { - finishSequence(); - return; + // `MultipartFile = [optional arguments] ^G` + else if (controlCommandPrefix.startsWith("MultipartFile=")) { + if (mITermImage != null) { + Logger.logWarn(mClient, LOG_TAG, "A new iTerm 'MultipartFile' command received while already processing a 'MultipartFile' command"); + mITermImage = null; // Unset old image. + } + + iTermImage = new ITermImage(mClient, /* multiPart */ true); + if (iTermImage.readArguments(this, mTerminalControlArgs, /* `1337;MultipartFile=` */ 19) < 19) { + iTermImage = null; + } else { + mITermImage = iTermImage; + } } - if (osc_colon >= 0 && mTerminalControlArgs.length() > osc_colon) { - while (mTerminalControlArgs.length() - osc_colon < 4) { - mTerminalControlArgs.append('='); + // `FilePart = base64 encoded file contents ^G` + else if (controlCommandPrefix.startsWith("FilePart=")) { + if (mITermImage == null) { + Logger.logError(mClient, LOG_TAG, "An iTerm 'FilePart' command received without a 'MultipartFile' command preceding it"); + return; } - try { - byte[] decoded = Base64.decode(mTerminalControlArgs.substring(osc_colon), 0); - for (int i = 0 ; i < decoded.length; i++) { - ESC_OSC_data.add(decoded[i]); - } - } catch(Exception e) { - // Ignore non-Base64 data. + + if (!mITermImage.readImage(mTerminalControlArgs, /* `1337;FilePart=` */ 14)) { + mITermImage = null; } - mTerminalControlArgs.setLength(osc_colon); } - if (osc_colon >= 0) { - byte[] result = new byte[ESC_OSC_data.size()]; - for(int i = 0; i < ESC_OSC_data.size(); i++) { - result[i] = ESC_OSC_data.get(i).byteValue(); + // `FileEnd ^G` + else if (controlCommandPrefix.equals("FileEnd")) { + if (mITermImage == null) { + Logger.logError(mClient, LOG_TAG, "An iTerm 'FileEnd' command received without a 'MultipartFile' command preceding it"); + return; } - int[] res = mScreen.addImage(result, mCursorRow, mCursorCol, cellW, cellH, width, height, aspect); - int col = res[1] + mCursorCol; - if (col < mColumns -1) { - res[0] -= 1; - } else { - col = 0; + + iTermImage = mITermImage; + mITermImage = null; // Free global reference so that memory is freed at end function. + if ( + !iTermImage.setMultiPartImageRead() || + !iTermImage.decodeImage()) { + iTermImage = null; } - for(;res[0] > 0; res[0]--) { - doLinefeed(); + } + + // Free image data from memory held in osc command arguments as it is no longer needed. + if (!oscArgsCleared) + clearTerminalControlArgs(); + + if (iTermImage != null && iTermImage.isImageDecoded()) { + // Display image as inline in Terminal. + if (iTermImage.isInline()) { + int[] cursorDelta = mScreen.addTerminalBitmapForImage(iTermImage.getDecodedImage(), + mCursorCol, mCursorRow, mCellWidthPixels, mCellHeightPixels, + iTermImage.getWidth(), iTermImage.getHeight(), + iTermImage.shouldPreserveAspectRatio()); + + int col = cursorDelta[1] + mCursorCol; + if (col < mColumns - 1) { + cursorDelta[0] -= 1; + } else { + col = 0; + } + for (; cursorDelta[0] > 0; cursorDelta[0]--) { + doLinefeed(); + } + mCursorCol = col; } - mCursorCol = col; - ESC_OSC_data.clear(); - } else { + // Saving files in downloads folder is not supported currently. + else {} } - } else if (textParameter.startsWith("ReportCellSize")) { - mSession.write(String.format(Locale.US, "\0331337;ReportCellSize=%d;%d\007", cellH, cellW)); + break; + } else if (controlCommandPrefix.startsWith("ReportCellSize")) { + mSession.write(String.format(Locale.ENGLISH, "\0331337;ReportCellSize=%d;%d\007", mCellHeightPixels, mCellWidthPixels)); } - break; + + // Free image from memory for any non `MultipartFile=` related commands. + mITermImage = null; default: unknownParameter(value); break; } + + // Free image from memory if an incomplete `MultipartFile` command was received without a `FileEnd`. + // The `mITermImage` cannot set to `null` in `clearOscTypeVariables()` as sequential + // OSC commands will be received for `MultipartFile` commands, and the variable is required + // to be set until the final `FileEnd` command is received. + if (mITermImage != null && value != 1337) { + mITermImage = null; + } + finishSequence(); } @@ -3129,12 +3427,9 @@ public void reset() { mColors.reset(); mSession.onColorsChanged(); - ESC_DCS_escape = false; - ESC_DCS_sixel = false; - ESC_OSC_colon = -1; - clearTerminalControlArgs(); clearOscTypeVariables(); + mITermImage = null; } public String getSelectedText(int x1, int y1, int x2, int y2) { diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java index a70103482f..14de528a07 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalRow.java @@ -49,8 +49,8 @@ public final class TerminalRow { final long[] mStyle; /** If this row might contain chars with width != 1, used for deactivating fast path */ boolean mHasNonOneWidthOrSurrogateChars; - /** If this row has a bitmap. Used for performace only */ - public boolean mHasBitmap; + /** If this row has a {@link TerminalBitmap}. Used for performance only. */ + public boolean mHasTerminalBitmap; /** Construct a blank row (containing only whitespace, ' ') with a specified style. */ public TerminalRow(int columns, long style) { @@ -148,7 +148,7 @@ public void clear(long style) { Arrays.fill(mStyle, style); mSpaceUsed = mColumns; mHasNonOneWidthOrSurrogateChars = false; - mHasBitmap = false; + mHasTerminalBitmap = false; } // https://github.com/steven676/Android-Terminal-Emulator/commit/9a47042620bec87617f0b4f5d50568535668fe26 @@ -158,8 +158,8 @@ public void setChar(int columnToSet, int codePoint, long style) { mStyle[columnToSet] = style; - if (!mHasBitmap && TextStyle.isBitmap(style)) { - mHasBitmap = true; + if (!mHasTerminalBitmap && TextStyle.isTerminalBitmap(style)) { + mHasTerminalBitmap = true; } final int newCodePointDisplayWidth = WcWidth.width(codePoint); diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java new file mode 100644 index 0000000000..d7f438a1fb --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java @@ -0,0 +1,289 @@ +package com.termux.terminal; + +import android.graphics.Bitmap; + +/** + * A terminal sixel received via `DCS q s..s ST` or `DCS P1; P2; P3; q s..s ST`. + * + * **See Also:** + * - `TerminalEmulator.ESC_DCS__SIXEL` + * - https://vt100.net/docs/vt3xx-gp/chapter14.html + * - https://en.wikipedia.org/wiki/Sixel + * - https://www.digiater.nl/openvms/decus/vax90b1/krypton-nasa/all-about-sixels.text + */ +public class TerminalSixel { + + public static final String LOG_TAG = "TerminalSixel"; + + + + public static final int[] SIXEL__INITIAL_COLOR_MAP = { + 0xFF000000, 0xFF3333CC, 0xFFCC2323, 0xFF33CC33, 0xFFCC33CC, 0xFF33CCCC, 0xFFCCCC33, 0xFF777777, + 0xFF444444, 0xFF565699, 0xFF994444, 0xFF569956, 0xFF995699, 0xFF569999, 0xFF999956, 0xFFCCCCCC + }; + + /** + * A sixel is a group of six pixels in a vertical column. + */ + public static final int SIXEL__LINE_LEN = 6; + + /** + * The max pixel dimensions of a sixel image bitmap. + * + * Each pixel is stored on 4 bytes for a {@link Bitmap.Config#ARGB_8888} bitmap color config, + * so a 2048x2048 sixel image will take 16,777,216 bytes/16MB. + */ + public static final int SIXEL__MAX_BITMAP_DIMENSION = 2048; + + public static final int SIXEL__BITMAP_RESIZE_EXTRA = 100; + + /** + * The max value for the sixel Graphics Repeat Introducer. + * + * Each repeat creates a new sixel line of `1x6` pixels, where `6` is the {@link #SIXEL__LINE_LEN}. + * + * - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.1 + */ + public static final int SIXEL__MAX_REPEAT = SIXEL__MAX_BITMAP_DIMENSION; + + + + protected final TerminalSessionClient mClient; + + protected Bitmap mBitmap; + + protected int mWidth; + protected int mHeight; + + protected int mCurX; + protected int mCurY; + + protected final int[] mColorMap; + protected int mColor; + + + + protected TerminalSixel(TerminalSessionClient client, Bitmap bitmap) { + mClient = client; + + mBitmap = bitmap; + + mWidth = 0; + mHeight = 0; + + mCurX = 0; + mCurY = 0; + + mColorMap = new int[256]; + System.arraycopy(SIXEL__INITIAL_COLOR_MAP, 0, mColorMap, 0, 16); + mColor = mColorMap[0]; + } + + + + public static TerminalSixel build(TerminalSessionClient client, int bitmapWidth, int bitmapHeight) { + try { + + Bitmap bitmap = Bitmap.createBitmap(bitmapWidth, bitmapHeight, Bitmap.Config.ARGB_8888); + bitmap.eraseColor(0); + + return new TerminalSixel(client, bitmap); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(client, LOG_TAG, "Create sixel bitmap for" + + " width " + bitmapWidth + " with height " + bitmapHeight + " failed: " + t.getMessage()); + return null; + } + } + + + + public TerminalSessionClient getClient() { + return mClient; + } + + + public Bitmap getBitmap() { + return mBitmap; + } + + + public int getWidth() { + return mWidth; + } + + public int getHeight() { + return mHeight; + } + + + public int getCurX() { + return mCurX; + } + + public int getCurY() { + return mCurY; + } + + + public int[] getColorMap() { + return mColorMap; + } + + public int getColor() { + return mColor; + } + + + + public boolean readData(int codePoint, int repeat) { + if (mBitmap == null) { + return false; + } + + // Graphics Carriage Return `$`. + // > The $ (2/4) character indicates the end of the sixel line. The active position returns + // > to the left page border of the same sixel line. You can use this character to overprint lines. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.4 + if (codePoint == '$') { + mCurX = 0; + return true; + } + + // Graphics New Line `-`. + // > The - (2/13) character indicates the end of a sixel line. The active position moves to + // > the left margin of the next sixel line. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.5 + if (codePoint == '-') { + mCurX = 0; + mCurY += SIXEL__LINE_LEN; + return true; + } + + if (mBitmap.getWidth() < mCurX + repeat) { + int newBitmapWidth = mCurX + repeat + SIXEL__BITMAP_RESIZE_EXTRA; + + if (newBitmapWidth < 0) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap width overflowed: " + mCurX + " (cursor x) + " + repeat + " (repeat) + " + SIXEL__BITMAP_RESIZE_EXTRA); + return false; + } + + if (newBitmapWidth > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap width " + newBitmapWidth + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + return false; + } + + mBitmap = TerminalBitmap.resizeBitmap(LOG_TAG, "sixel", mClient, mBitmap, newBitmapWidth, mBitmap.getHeight()); + if (mBitmap == null) { + return false; + } + } + + if (mBitmap.getHeight() < mCurY + SIXEL__LINE_LEN) { + // Very unlikely to resize both at the same time. + int newBitmapHeight = mCurY + SIXEL__BITMAP_RESIZE_EXTRA; + + if (newBitmapHeight < 0) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap height overflowed: " + mCurY + " (cursor y) + " + SIXEL__BITMAP_RESIZE_EXTRA); + return false; + } + + if (newBitmapHeight > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap height " + newBitmapHeight + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + return false; + } + + mBitmap = TerminalBitmap.resizeBitmap(LOG_TAG, "sixel", mClient, mBitmap, mBitmap.getWidth(), newBitmapHeight); + if (mBitmap == null) { + return false; + } + } + + if (mCurX + repeat > mBitmap.getWidth()) { + repeat = mBitmap.getWidth() - mCurX; + } + + if (mCurY + SIXEL__LINE_LEN > mBitmap.getHeight()) { + Logger.logError(mClient, LOG_TAG, "The sixel curson y position " + + mCurY + SIXEL__LINE_LEN + " is greater than bitmap height " + mBitmap.getHeight()); + return false; + } + + // Sixel data characters are in the range of `?` (0x3F) to `~` (0x7E). + // > Each sixel data character represents six vertical pixels of data. Each sixel data character + // > represents a binary value equal to the character code value minus hex 3F. + // - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.2.1 + if (repeat > 0 && codePoint >= '?' && codePoint <= '~') { + int b = codePoint - '?'; + if (mCurY + SIXEL__LINE_LEN > mHeight) { + mHeight = mCurY + SIXEL__LINE_LEN; + } + + while (repeat-- > 0) { + for (int i = 0; i < SIXEL__LINE_LEN; i++) { + if ((b & (1 << i)) != 0) { + mBitmap.setPixel(mCurX, mCurY + i, mColor); + } + } + + mCurX += 1; + if (mCurX > mWidth) { + mWidth = mCurX; + } + } + } + + return true; + } + + public boolean resize(int sixelWidth, int sixelHeight) { + if (mBitmap == null) { + return false; + } + + if (sixelWidth < 1 || sixelHeight < 1) + return false; + + int bitmapWidth = mBitmap.getWidth(); + int newBitmapWidth = Math.max(sixelWidth, bitmapWidth); + + int bitmapHeight = mBitmap.getHeight(); + int newBitmapHeight = Math.max(sixelHeight, bitmapHeight); + + if (bitmapWidth < newBitmapWidth || bitmapHeight < newBitmapHeight) { + if (newBitmapWidth > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap resize width " + newBitmapWidth + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + return false; + } + + if (newBitmapHeight > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap resize height " + newBitmapHeight + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + return false; + } + + mBitmap = TerminalBitmap.resizeBitmap(LOG_TAG, "sixel", mClient, mBitmap, newBitmapWidth, newBitmapHeight); + if (mBitmap == null) { + return false; + } + } + + return true; + } + + public void setColor(int color) { + if (color >= 0 && color < mColorMap.length) { + mColor = mColorMap[color]; + } + } + + public void setRGBColor(int color, int r, int g, int b) { + if (color >= 0 && color < mColorMap.length) { + int red = Math.min(255, r * 255/100); + int green = Math.min(255, g * 255/100); + int blue = Math.min(255, b * 255/100); + mColor = 0xff000000 + (red << 16) + (green << 8) + blue; + mColorMap[color] = mColor; + } + } + +} diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java b/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java index 7ee4b06ebc..008d1e777b 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TextStyle.java @@ -35,8 +35,8 @@ public final class TextStyle { private final static int CHARACTER_ATTRIBUTE_TRUECOLOR_FOREGROUND = 1 << 9; /** If true (24-bit) color is used for the cell for foreground. */ private final static int CHARACTER_ATTRIBUTE_TRUECOLOR_BACKGROUND= 1 << 10; - /** If true, character represents a bitmap slice, not text. */ - public final static int BITMAP = 1 << 15; + /** If true, character represents a {@link TerminalBitmap} slice, not text. */ + public final static int TERMINAL_BITMAP = 1 << 15; public final static int COLOR_INDEX_FOREGROUND = 256; public final static int COLOR_INDEX_BACKGROUND = 257; @@ -89,24 +89,28 @@ public static int decodeEffect(long style) { return (int) (style & 0b11111111111); } - public static long encodeBitmap(int num, int X, int Y) { - return ((long)num << 16) | ((long)Y << 32) | ((long)X << 48) | BITMAP; - } - public static boolean isBitmap(long style) { - return (style & 0x8000) != 0; + + public static long encodeTerminalBitmap(int num, int x, int y) { + return ((long) x << 48) | ((long) y << 32) | ((long) num << 16) | TERMINAL_BITMAP; } - public static int bitmapNum(long style) { - return (int)(style & 0xffff0000) >> 16; + public static boolean isTerminalBitmap(long style) { + return (style & TERMINAL_BITMAP) != 0; + } + + /* The bitmap num, x or y could have value `0`, so only return value (especially `0`) if bitmap bit is set. */ + + public static int getTerminalBitmapNum(long style) { + return (style & TERMINAL_BITMAP) != 0 ? (int) (style & 0xffff0000L) >> 16 : -1; } - public static int bitmapX(long style) { - return (int)((style >> 48) & 0xfff); + public static int getTerminalBitmapX(long style) { + return (style & TERMINAL_BITMAP) != 0 ? (int) ((style >> 48) & 0xfff) : -1; } - public static int bitmapY(long style) { - return (int)((style >> 32) & 0xfff); + public static int getTerminalBitmapY(long style) { + return (style & TERMINAL_BITMAP) != 0 ? (int) ((style >> 32) & 0xfff) : -1; } } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java deleted file mode 100644 index e9f0e2ed05..0000000000 --- a/terminal-emulator/src/main/java/com/termux/terminal/WorkingTerminalBitmap.java +++ /dev/null @@ -1,110 +0,0 @@ -package com.termux.terminal; - -import android.graphics.Bitmap; - -/** - * A circular buffer of {@link TerminalRow}:s which keeps notes about what is visible on a logical screen and the scroll - * history. - *

    - * See {@link #externalToInternalRow(int)} for how to map from logical screen rows to array indices. - */ -public final class WorkingTerminalBitmap { - final private int sixelInitialColorMap[] = {0xFF000000, 0xFF3333CC, 0xFFCC2323, 0xFF33CC33, 0xFFCC33CC, 0xFF33CCCC, 0xFFCCCC33, 0xFF777777, - 0xFF444444, 0xFF565699, 0xFF994444, 0xFF569956, 0xFF995699, 0xFF569999, 0xFF999956, 0xFFCCCCCC}; - private int[] colorMap; - private int curX; - private int curY; - private int color; - public int width; - public int height; - public Bitmap bitmap; - private static final String LOG_TAG = "WorkingTerminalBitmap"; - - public WorkingTerminalBitmap(int w, int h) { - try { - bitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); - } catch (OutOfMemoryError e) { - Logger.logWarn(null, LOG_TAG, "Out of memory - sixel ignored"); - bitmap = null; - } - bitmap.eraseColor(0); - width = 0; - height = 0; - curX = 0; - curY = 0; - colorMap = new int[256]; - for (int i=0; i<16; i++) { - colorMap[i] = sixelInitialColorMap[i]; - } - color = colorMap[0]; - } - - public void sixelChar(int c, int rep) { - if (bitmap == null) { - return; - } - if (c == '$') { - curX = 0; - return; - } - if (c == '-') { - curX = 0; - curY += 6; - return; - } - if (bitmap.getWidth() < curX + rep) { - try { - bitmap = TerminalBitmap.resizeBitmap(bitmap, curX + rep + 100, bitmap.getHeight()); - } catch(OutOfMemoryError e) { - Logger.logWarn(null, LOG_TAG, "Out of memory - sixel truncated"); - } - } - if (bitmap.getHeight() < curY + 6) { - // Very unlikely to resize both at the same time - try { - bitmap = TerminalBitmap.resizeBitmap(bitmap, bitmap.getWidth(), curY + 100); - } catch(OutOfMemoryError e) { - Logger.logWarn(null, LOG_TAG, "Out of memory - sixel truncated"); - } - } - if (curX + rep > bitmap.getWidth()) { - rep = bitmap.getWidth() - curX; - } - if ( curY + 6 > bitmap.getHeight()) { - return; - } - if (rep > 0 && c >= '?' && c <= '~') { - int b = c - '?'; - if (curY + 6 > height) { - height = curY + 6; - } - while (rep-- > 0) { - for (int i = 0 ; i < 6 ; i++) { - if ((b & (1< width) { - width = curX; - } - } - } - } - - public void sixelSetColor(int col) { - if (col >= 0 && col < 256) { - color = colorMap[col]; - } - } - - public void sixelSetColor(int col, int r, int g, int b) { - if (col >= 0 && col < 256) { - int red = Math.min(255, r*255/100); - int green = Math.min(255, g*255/100); - int blue = Math.min(255, b*255/100); - color = 0xff000000 + (red << 16) + (green << 8) + blue; - colorMap[col] = color; - } - } -} diff --git a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java index 2be86f17fa..787902ce08 100644 --- a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java +++ b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java @@ -69,7 +69,6 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final TerminalBuffer screen = mEmulator.getScreen(); final int[] palette = mEmulator.mColors.mCurrentColors; final int cursorShape = mEmulator.getCursorStyle(); - mEmulator.setCellSize((int)mFontWidth, (int)mFontLineSpacing); if (reverseVideo) canvas.drawColor(palette[TextStyle.COLOR_INDEX_FOREGROUND], PorterDuff.Mode.SRC); @@ -104,13 +103,14 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final int charsForCodePoint = charIsHighsurrogate ? 2 : 1; final int codePoint = charIsHighsurrogate ? Character.toCodePoint(charAtIndex, line[currentCharIndex + 1]) : charAtIndex; final long style = lineObject.getStyle(column); - if (TextStyle.isBitmap(style)) { - Bitmap bm = mEmulator.getScreen().getSixelBitmap(codePoint, style); - if (bm != null) { + if (TextStyle.isTerminalBitmap(style)) { + Bitmap bitmap = mEmulator.getScreen().getSixelBitmap(style); + if (bitmap != null) { float left = column * mFontWidth; float top = heightOffset - mFontLineSpacing; - RectF r = new RectF(left, top, left + mFontWidth, top + mFontLineSpacing); - canvas.drawBitmap(mEmulator.getScreen().getSixelBitmap(codePoint, style), mEmulator.getScreen().getSixelRect(codePoint, style), r, null); + Rect bitmapSrcRect = mEmulator.getScreen().getSixelRect(style); + RectF bitmapDestRect = new RectF(left, top, left + mFontWidth, top + mFontLineSpacing); + canvas.drawBitmap(bitmap, bitmapSrcRect, bitmapDestRect, null); } column += 1; measuredWidthForRun = 0.f; From 3ec521f6046688c280a2a3c50763b5b19823a78e Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Tue, 14 Apr 2026 01:17:12 +0500 Subject: [PATCH 09/15] Changed: Increase Sixel and iTerm Image limits The `TerminalBitmap.MAX_BITMAP_SIZE` defines the max size of a Terminal bitmap for its pixels. Each pixel is stored on 4 bytes for a `Bitmap.Config.ARGB_8888` bitmap color config. The value should normally be between `100-200MB` depending on device and Android version. Check the variable docs for more info. The sixel image size cannot be greater than `TerminalBitmap.MAX_BITMAP_SIZE`. The repeat value for sixel Graphics Repeat Introducer command cannot be greater than `TerminalSixel.SIXEL__MAX_REPEAT` (`8192`). The iTerm image data sent for `File=` and `MultipartFile=` protocols cannot be greater than `TerminalBitmap.MAX_BITMAP_SIZE` bytes. --- .../com/termux/terminal/AndroidUtils.java | 65 +++++++++ .../com/termux/terminal/TerminalBitmap.java | 124 +++++++++++++++++- .../com/termux/terminal/TerminalBuffer.java | 16 ++- .../com/termux/terminal/TerminalEmulator.java | 124 +++++++++++++----- .../com/termux/terminal/TerminalSixel.java | 52 +++----- 5 files changed, 310 insertions(+), 71 deletions(-) create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/AndroidUtils.java diff --git a/terminal-emulator/src/main/java/com/termux/terminal/AndroidUtils.java b/terminal-emulator/src/main/java/com/termux/terminal/AndroidUtils.java new file mode 100644 index 0000000000..2f762c7bbc --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/AndroidUtils.java @@ -0,0 +1,65 @@ +package com.termux.terminal; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AndroidUtils { + + /** + * Get system properties as {@link Properties} from `/system/bin/getprop` command output + * that reads the `/system/build.prop` file. + * + * Sourced from https://github.com/termux/termux-app/blob/30ebb2dee381d292ade0f2868cfde0f9f20b89fe/termux-shared/src/main/java/com/termux/shared/android/AndroidUtils.java#L170. + */ + public static Properties getSystemProperties(String logTag) { + Properties systemProperties = new Properties(); + + // getprop commands returns values in the format `[key]: [value]` + // Regex matches string starting with a literal `[`, + // followed by one or more characters that do not match a closing square bracket as the key, + // followed by a literal `]: [`, + // followed by one or more characters as the value, + // followed by string ending with literal `]` + // multiline values will be ignored + Pattern propertiesPattern = Pattern.compile("^\\[([^]]+)]: \\[(.+)]$"); + + try { + Process process = new ProcessBuilder() + .command("/system/bin/getprop") + .redirectErrorStream(true) + .start(); + + InputStream inputStream = process.getInputStream(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + String line, key, value; + + while ((line = bufferedReader.readLine()) != null) { + Matcher matcher = propertiesPattern.matcher(line); + if (matcher.matches()) { + key = matcher.group(1); + value = matcher.group(2); + if (key != null && value != null && !key.isEmpty() && !value.isEmpty()) + systemProperties.put(key, value); + } + } + + bufferedReader.close(); + process.destroy(); + + } catch (IOException e) { + Logger.logStackTraceWithMessage(null, logTag, "Failed to get run \"/system/bin/getprop\" to get system properties.", e); + } + + //for (String key : systemProperties.stringPropertyNames()) { + // Logger.logVerbose(null, logTag, key + ": " + systemProperties.get(key)); + //} + + return systemProperties; + } + +} diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java index 38eccad1f8..188f06bc2a 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java @@ -1,7 +1,16 @@ package com.termux.terminal; +import android.app.WallpaperManager; import android.graphics.Bitmap; import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.RecordingCanvas; +import android.graphics.Rect; +import android.graphics.RectF; +import android.os.Build; + +import java.util.Properties; /** * A terminal bitmap for images. @@ -10,6 +19,112 @@ public class TerminalBitmap { public static final String LOG_TAG = "TerminalBitmap"; + private static int initMaxBitmapSize() { + // Synced with `RecordingCanvas.MAX_BITMAP_SIZE`. + // - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/graphics/java/android/graphics/RecordingCanvas.java;l=42-50 + int defaultSize = + Build.VERSION.SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM ? + 150 * 1024 * 1024 : // 150 MB + 100 * 1024 * 1024; // 100 MB + + Properties systemProperties = AndroidUtils.getSystemProperties(LOG_TAG); + String maxTextureSizeString = systemProperties.getProperty("ro.hwui.max_texture_allocation_size"); + + if (maxTextureSizeString == null) return defaultSize; + + try { + int maxTextureSize = Integer.parseInt(maxTextureSizeString); + return maxTextureSize > 0 ? maxTextureSize : defaultSize; + } + catch (Exception e) { + return defaultSize; + } + } + + /** + * The max size of a Terminal {@link Bitmap} for its pixels. The limit is defined as per how + * `RecordingCanvas.MAX_BITMAP_SIZE` value is defined, check below for details. The value should + * normally be between `100-200MB` depending on device and Android version. + * + * Each pixel is stored on 4 bytes for a {@link Bitmap.Config#ARGB_8888} bitmap color config. + * The bitmap will have following memory usage for its respective resolution (`width x height x 4`). + * - 1280x720 (HD): 3,686,400 bytes/3.6MB. + * - 1920x1080 (FHD): 8,294,400 bytes/8MB. + * - 2560x1440 (QHD): 14,745,600 bytes/14.7MB. + * - 3840x2160 (4K UHD): 33,177,600 bytes/33MB. + * - 7680x4320 (8K UHD): 132,710,400 bytes/132MB. + * . + * - https://en.wikipedia.org/wiki/Display_resolution_standards#High-definition + * + * The terminal uses {@link Canvas#drawBitmap(Bitmap, Rect, RectF, Paint)} to draw the bitmap + * when `TerminalRenderer.render()` is called. + * + * The {@link Canvas} class defines `Canvas.MAXIMUM_BITMAP_SIZE` for the maximum dimension + * for a bitmap which is returned by {@link Canvas#getMaximumBitmapWidth()} and + * {@link Canvas#getMaximumBitmapHeight()}. It is hardcoded with the value `32766` as defined by + * Skia (2D graphics library), which technically has the limit `32767` as it requires supporting + * math on 16-bit buffers. + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/f61970fc79e9c5cf340fa942597628242361864a + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/graphics/java/android/graphics/Canvas.java;l=76-78 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:external/skia/src/shaders/SkImageShader.cpp;l=254-267 + * + * The {@link RecordingCanvas} class defines `RecordingCanvas.MAX_BITMAP_SIZE` for the + * maximum size (not dimension) for a bitmap, which is checked by + * `RecordingCanvas.throwIfCannotDraw()` when `BaseRecordingCanvas.drawBitmap()` is called. + * The `RecordingCanvas` is a specialized implementation of the `Canvas` class that is designed + * to record draw commands for deferred rendering instead of executing draw commands instantly. + * By recording draw commands, they can be cached so that complex views can be efficiently + * re-drawn without recalculating them again for every frame. The caching part is similar to + * how a terminal behaves, where it stores all the bitmaps for rendering depending on scroll + * position. So both `RecordingCanvas` and a terminal require similar limits on bitmap + * sizes considering memory consumption limits of apps, and multiple bitmaps being loaded + * instead of a single one like for wallpapers, hence why `TerminalBitmap.MAX_BITMAP_SIZE` is + * synced with `RecordingCanvas`. + * The `RecordingCanvas.MAX_BITMAP_SIZE` is set from `ro.hwui.max_texture_allocation_size` + * system property if set for Android `>= 12`, otherwise `150MB` (`100MB` for Android `10-14`). + * The values `>= 150MB` are enough to support `7680x4320` (8K UHD) bitmaps. + * Some devices like larger xiaomi devices have `ro.hwui.max_texture_allocation_size` set to `209715200` (`200MB`). + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/e4d011201cea40d46cb2b2eef401db8fddc5c9c6 + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/0e717a9d06ded980908649393bd73e46ffafcd54 + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/97396260ed06cc9d1834d4d8e4e649a3ef09f1f3 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/graphics/java/android/graphics/RecordingCanvas.java;l=42-50 + * + * The Android wallpaper manager service also checks if dimensions of cropped wallpaper exceeds + * max texture size that the GPU can support, otherwise it will cause System UI to keep crashing + * because it can not initialize EGL with an appropriate surface. The `GLHelper.getMaxTextureSize()` + * returns the max texture size, which is defined by `sys.max_texture_size` system property if set, + * otherwise by value for `GL_MAX_TEXTURE_SIZE`. The `sys.max_texture_size` defines the maximum + * width or height of a texture, not total size. Its value can be low like `2048` or high like + * `16384` for 16K support. + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/32c6a7c691b0d91085c1ed13fe6f1c473c94b4c8 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/services/core/java/com/android/server/wallpaper/WallpaperCropper.java;l=461 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/services/core/java/com/android/server/wallpaper/GLHelper.java;l=145 + * - https://developer.android.com/reference/android/opengl/GLES10#GL_MAX_TEXTURE_SIZE + * + * The {@link WallpaperManager#getDesiredMinimumWidth()} and {@link WallpaperManager#getDesiredMinimumHeight()} + * can also be called to get minimum suggested width and height of the wallpaper that an app + * should use when setting the wallpaper. This normally is equal to the width and height of the + * current device display, but the width can be higher than display width if the homescreen is + * scrollable horizontally with multiple pages, in which case the width returned is equal to + * entire workspace width. The launcher apps can provide Android their desired width and height + * dimensions depending on the homescreen pages config by calling + * {@link WallpaperManager#suggestDesiredDimensions(int, int)}, which also ensures that values + * passed are scaled down to `sys.max_texture_size` system property if its set. + * - https://cs.android.com/android/_/android/platform/frameworks/base/+/289c273ec49462c7bfdbf6238e9016936da7307c + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/core/java/android/app/WallpaperManager.java;l=2737-2794 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/services/core/java/com/android/server/wallpaper/WallpaperManagerService.java;l=2330-2366 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/services/core/java/com/android/server/wallpaper/WallpaperDisplayHelper.java;l=108-115 + * - https://cs.android.com/android/platform/superproject/+/android-16.0.0_r1:frameworks/base/core/java/android/view/Display.java;l=1052-1063 + * + * If an app specifies `largeHeap=true` in its `AndroidManifest.xml`, then it can be allocated + * larger heap memory to load larger bitmaps maps instead of resulting in an OOM. The Termux app + * does not have it enabled, and hence is more likely to have OOMs when loading larger bitmaps. + * - https://developer.android.com/guide/topics/manifest/application-element#largeHeap + * - https://developer.android.com/topic/performance/memory + */ + public static final int MAX_BITMAP_SIZE = initMaxBitmapSize(); + + protected final TerminalSessionClient mClient; @@ -315,8 +430,15 @@ public static Bitmap resizeBitmap(String logTag, String label, TerminalSessionCl Bitmap newBitmap; try { - int[] pixels = new int[bitmap.getAllocationByteCount()]; + int newBitmapSize = bitmapWidth * bitmapHeight * 4; + if (newBitmapSize < 0 || newBitmapSize > MAX_BITMAP_SIZE) { + Logger.logError(client, logTag, "The new " + label + " bitmap after resize with" + + " width " + bitmapWidth + " and height " + bitmapHeight + + " has size " + newBitmapSize + " greater than max bitmap size " + MAX_BITMAP_SIZE); + return null; + } + int[] pixels = new int[bitmap.getAllocationByteCount()]; bitmap.getPixels(pixels, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight()); newBitmap = Bitmap.createBitmap(bitmapWidth, bitmapHeight, Bitmap.Config.ARGB_8888); diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java index fea81faf41..26ce7120c5 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java @@ -630,13 +630,22 @@ public synchronized void sixelClear() { mTerminalSixel = null; } + /** + * Clears the {@link #mTerminalSixel} by setting it to `null` and logs error. + * Call this on error if further sixel commands/data should be parsed to prevent them from + * printing on terminal, but sixel rendering should be ignored. + */ + public synchronized void sixelIgnore() { + Logger.logError(mClient, LOG_TAG, "Ignoring sixel rendering"); + mTerminalSixel = null; + } + public synchronized boolean sixelReadData(int codePoint, int repeat) { // If an error occurred during processing (like OOM), then remaining sixel command is // completely read, but is ignored. if (mTerminalSixel != null) { if (!mTerminalSixel.readData(codePoint, repeat)) { - // Ignore further commands/data. - mTerminalSixel = null; + sixelIgnore(); return false; } } @@ -648,8 +657,7 @@ public synchronized boolean sixelResize(int sixelWidth, int sixelHeight) { // completely read, but is ignored. if (mTerminalSixel != null) { if (!mTerminalSixel.resize(sixelWidth, sixelHeight)) { - // Ignore further commands/data. - mTerminalSixel = null; + sixelIgnore(); return false; } } diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index 64473e019f..aada13b78f 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -274,10 +274,15 @@ public final class TerminalEmulator { private static final int TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY = 16; /** - * The max length for {@link #mTerminalControlArgs}. + * The default max length for {@link #mTerminalControlArgs}. * Needs to be large enough to contain reasonable OSC 52 pastes, sixel and iterm images data. */ - private static final int TERMINAL_CONTROL_ARGS__MAX_LENGTH = 16384; + private static final int TERMINAL_CONTROL_ARGS__DEFAULT_MAX_LENGTH = 16384; + + /** + * The max length for {@link #mTerminalControlArgs} used by {@link #collectTerminalControlArgs(int)}. + */ + private int mTerminalControlArgsMaxLength = TERMINAL_CONTROL_ARGS__DEFAULT_MAX_LENGTH; /** The terminal control arguments string buffer, like for OSC, DCS, APC commands. */ private StringBuilder mTerminalControlArgs = new StringBuilder(TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY); @@ -1145,7 +1150,7 @@ private void doDcs(final int b) { // command, then it is added to buffer in code below and more input is waited for as // further arguments need to be received for its command before it can be processed, // which is not until the next command is received. - // We wait till at least `TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2` commands string has + // We wait till at least `mTerminalControlArgsMaxLength / 2` commands string has // been received. The divide by 2 is done since if near the max length, a new command // starts and it does not end before the max length, then `Terminal control args overflow // error would occur. @@ -1156,7 +1161,7 @@ private void doDcs(final int b) { // `mTerminalControlArgs.length() > 1` is done so that loop does not engage on first // character after `q` and only after first command has been fully received. (ESC_DCS__SIXEL && ((b == '#' || b == '!' || b == '"') && - ((mTerminalControlArgs.length() >= (TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2)) || (mTerminalControlArgs.length() > 1 && mSixelCommandPartNum == 1)))) + ((mTerminalControlArgs.length() >= (mTerminalControlArgsMaxLength / 2)) || (mTerminalControlArgs.length() > 1 && mSixelCommandPartNum == 1)))) ) { String dcs = mTerminalControlArgs.toString(); @@ -1411,6 +1416,9 @@ private boolean processSixelDcs(String dcs) { int color; boolean isValidDcs = true; + // FIXME: Use `StringUtils.truncateLogStringWithSnippet()` when its added for logging `dcs` input + // in errors as it may be truncated from end before index if error length is greater than + // `Logger.LOGGER_ENTRY_MAX_SAFE_PAYLOAD`. while (index < dcs.length()) { ch = dcs.charAt(index); @@ -1439,8 +1447,8 @@ else if (ch == '#') { ch = dcs.charAt(index); if (ch >= '0' && ch <= '9') { color = color * 10 + ch - '0'; - if (color > 255) { - Logger.logError(mClient, LOG_TAG, "The sixel color command Pc value " + color + " is not between 0-255 at index " + index + " of sixel input: " + dcs); + if (color < 0) { // Overflow. + Logger.logError(mClient, LOG_TAG, "The sixel color command Pc value overflow at index " + index + " of sixel input: " + dcs); isValidDcs = false; break; } @@ -1450,6 +1458,12 @@ else if (ch == '#') { } } + if (color > 255) { + Logger.logError(mClient, LOG_TAG, "The sixel color command Pc value " + color + " is not between 0-255 at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + if (!isValidDcs) { break; } @@ -1471,22 +1485,12 @@ else if (ch == '#') { if (ch >= '0' && ch <= '9') { if (incArg) { arg++; incArg = false; } args[arg] = args[arg] * 10 + ch - '0'; - if (arg == 0) { // Pu must equal 1 or 2. - if ((args[arg] != 1 && args[arg] != 2)) { - Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command Pu value " + args[arg] + " is not 1 or 2 at index " + index + " of sixel input: " + dcs); - isValidDcs = false; - break; - } - } else { - int limit = 100; - if (args[0] == 1 && arg == 1) limit = 360; - if (args[arg] > limit) { - String argName = ""; - switch (arg) { case 1: argName = "pX"; break; case 2: argName = "pY"; break; case 3: argName = "pZ"; break; } - Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command " + argName + " value " + args[arg] + " is not between 0-" + limit + " at index " + index + " of sixel input: " + dcs); - isValidDcs = false; - break; - } + if (args[arg] < 0) { // Overflow. + String argName = ""; + switch (arg) { case 0: argName = "Pu"; break; case 1: argName = "pX"; break; case 2: argName = "pY"; break; case 3: argName = "pZ"; break; } + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command " + argName + " value overflow at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; } } else if (ch == ';') { if (arg == 3) { // Pz must not end with a ';'. @@ -1502,9 +1506,37 @@ else if (ch == '#') { index++; } + if (!isValidDcs) { + break; + } + + for (int i = 0; i < args.length; i++) { + if (i == 0) { // Pu must equal 1 or 2. + if ((args[i] != 1 && args[i] != 2)) { + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command Pu value " + args[i] + " is not 1 or 2 at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } else { + int limit = 100; + if (args[0] == 1 && i == 1) limit = 360; + if (args[i] < 0 || args[i] > limit) { + String argName = ""; + switch (i) { case 1: argName = "pX"; break; case 2: argName = "pY"; break; case 3: argName = "pZ"; break; } + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command " + argName + " value " + args[i] + " is not between 0-" + limit + " at index " + index + " of sixel input: " + dcs); + isValidDcs = false; + break; + } + } + } + if (isValidDcs && arg == 3) { // If complete spec is received and is valid. if (args[0] == 2) { // Only RGB is supported. mScreen.sixelSetRGBColor(color, args[1], args[2], args[3]); + } else if (args[0] == 1) { // HLS is not supported. + Logger.logError(mClient, LOG_TAG, "The sixel non-basic color command Pu value " + args[0] + " is not supported at index " + index + " of sixel input: " + dcs); + mScreen.sixelIgnore(); + break; } } else { if (isValidDcs) @@ -1524,9 +1556,8 @@ else if (ch == '!') { ch = dcs.charAt(index); if (ch >= '0' && ch <= '9') { repeat = repeat * 10 + ch - '0'; - if (repeat > TerminalSixel.SIXEL__MAX_REPEAT) { - Logger.logError(mClient, LOG_TAG, "The sixel repeat command Pn value " + repeat + " is greater than max repeat value " + - TerminalSixel.SIXEL__MAX_REPEAT + " at index " + index + " of sixel input: " + dcs); + if (repeat < 0) { // Overflow. + Logger.logError(mClient, LOG_TAG, "The sixel repeat command Pn value overflow at index " + index + " of sixel input: " + dcs); isValidDcs = false; break; } @@ -1536,6 +1567,13 @@ else if (ch == '!') { } } + if (repeat > TerminalSixel.SIXEL__MAX_REPEAT) { + Logger.logError(mClient, LOG_TAG, "The sixel repeat command Pn value " + repeat + " is greater than max repeat value " + + TerminalSixel.SIXEL__MAX_REPEAT + " at index " + index + " of sixel input: " + dcs); + mScreen.sixelIgnore(); + break; + } + if (!isValidDcs) { break; } @@ -1586,9 +1624,9 @@ else if (ch == '"') { // 2% extra for sixel commands/parameters in addition to image data. int sixelArgsExpectedLength = (int) (sixelWidth * sixelHeight * 1.02); // If sixel commands are too long, they are divided into parts, and if a - // new command starts near `TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2`, it could + // new command starts near `mTerminalControlArgsMaxLength / 2`, it could // contain image data for 1 pixel line of image width, so add that. - int sixelArgsPartsExpectedLength = (int) ((((double) TERMINAL_CONTROL_ARGS__MAX_LENGTH / 2) + sixelWidth) * 1.02); + int sixelArgsPartsExpectedLength = (int) ((((double) mTerminalControlArgsMaxLength / 2) + sixelWidth) * 1.02); int sixelArgsExpectedCapacity = Math.min(sixelArgsPartsExpectedLength, sixelArgsExpectedLength); if (sixelArgsExpectedCapacity > SIXEL_ARGS__INITIAL_CAPACITY) { mSixelArgsCapacity = sixelArgsExpectedCapacity; @@ -2654,6 +2692,7 @@ void setOscTypeVariables() { if (mOscType >= 0) { Integer terminalControlArgsCapacity = null; + Integer terminalControlArgsMaxLength = null; switch (mOscType) { case 1337: // iTerm image command sends the base64 encoded image, do not run complex logic for each byte. mIsFastPathOsc = true; @@ -2664,12 +2703,18 @@ void setOscTypeVariables() { // > Older versions of tmux have a limit of 256 bytes for the entire sequence. // - https://iterm2.com/documentation-images.html terminalControlArgsCapacity = 256; + terminalControlArgsMaxLength = TerminalBitmap.MAX_BITMAP_SIZE + + /* `1337;File=inline=1;size=209715200;name=xxxxxxxxxxxxxxxxxxxxxxxx;width=8192px;height=8192px;preserveAspectRatio=1:` */ 150; break; } if (terminalControlArgsCapacity != null) { ensureTerminalControlArgsCapacity(terminalControlArgsCapacity); } + + if (terminalControlArgsMaxLength != null) { + setTerminalControlArgsMaxLength(terminalControlArgsMaxLength); + } } } @@ -3085,7 +3130,9 @@ private int getArg(int index, int defaultValue, boolean treatZeroAsDefault) { /** Collect code point in {@link #mTerminalControlArgs}. */ private boolean collectTerminalControlArgs(int b) { - if (mTerminalControlArgs.length() < TERMINAL_CONTROL_ARGS__MAX_LENGTH) { + // FIXME: Use `Logger.logErrorDebug()` and elsewhere in terminal code when support is added + // to prevent logging potentially private data to logcat unless user has increased log level. + if (mTerminalControlArgs.length() < mTerminalControlArgsMaxLength) { try { // Appending can cause an increase in capacity and cause an OOM. mTerminalControlArgs.appendCodePoint(b); @@ -3095,13 +3142,14 @@ private boolean collectTerminalControlArgs(int b) { if (t instanceof OutOfMemoryError) System.gc(); Logger.logError(mClient, LOG_TAG, "Terminal control args collect failed for" + " char '" + (char) b + "' (numeric value=" + b + ") and" + - " args string '" + mTerminalControlArgs.substring(0, Math.min(16, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length() + + " args string '" + mTerminalControlArgs.substring(0, Math.min(100, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length() + ": " + t.getMessage()); } } else { - Logger.logError(mClient, LOG_TAG, "Terminal control args overflow for" + + Logger.logError(mClient, LOG_TAG, "Terminal control args input will" + + " overflow max args length " + mTerminalControlArgsMaxLength + " for" + " char '" + (char) b + "' (numeric value=" + b + ") and" + - " args string '" + mTerminalControlArgs.substring(0, Math.min(16, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length()); + " args string '" + mTerminalControlArgs.substring(0, Math.min(100, mTerminalControlArgs.length())) + "...' with length " + mTerminalControlArgs.length()); } clearTerminalControlArgs(); @@ -3111,6 +3159,8 @@ private boolean collectTerminalControlArgs(int b) { /** Clear {@link #mTerminalControlArgs}. */ private void clearTerminalControlArgs() { + mTerminalControlArgsMaxLength = TERMINAL_CONTROL_ARGS__DEFAULT_MAX_LENGTH; + if (mTerminalControlArgs.capacity() <= TERMINAL_CONTROL_ARGS__INITIAL_CAPACITY) { // Mark existing buffer as empty and reuse old array already allocated in // `StringBuffer` for future commands if required. @@ -3149,6 +3199,18 @@ private void ensureTerminalControlArgsCapacity(int capacity) { mTerminalControlArgs.ensureCapacity(capacity); } + /** + * Set {@link #mTerminalControlArgsMaxLength} in case a command expects a larger input. + * + * @param length The new max length. + */ + private void setTerminalControlArgsMaxLength(int length) { + if (length > 0) { + mTerminalControlArgsMaxLength = length; + } + } + + diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java index d7f438a1fb..fa1340b4b6 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalSixel.java @@ -28,23 +28,20 @@ public class TerminalSixel { public static final int SIXEL__LINE_LEN = 6; /** - * The max pixel dimensions of a sixel image bitmap. - * - * Each pixel is stored on 4 bytes for a {@link Bitmap.Config#ARGB_8888} bitmap color config, - * so a 2048x2048 sixel image will take 16,777,216 bytes/16MB. + * The amount of extra dimension added when resizing a sixel when receiving more image data. */ - public static final int SIXEL__MAX_BITMAP_DIMENSION = 2048; - - public static final int SIXEL__BITMAP_RESIZE_EXTRA = 100; + public static final int SIXEL__BITMAP_RESIZE_EXTRA_DIMENSION = 100; /** * The max value for the sixel Graphics Repeat Introducer. * + * The value `8192` can support 8K images, see also {@link TerminalBitmap#MAX_BITMAP_SIZE}. + * * Each repeat creates a new sixel line of `1x6` pixels, where `6` is the {@link #SIXEL__LINE_LEN}. * * - https://vt100.net/docs/vt3xx-gp/chapter14.html#S14.3.1 */ - public static final int SIXEL__MAX_REPEAT = SIXEL__MAX_BITMAP_DIMENSION; + public static final int SIXEL__MAX_REPEAT = 8192; @@ -141,6 +138,11 @@ public boolean readData(int codePoint, int repeat) { return false; } + if (repeat > TerminalSixel.SIXEL__MAX_REPEAT) { + Logger.logError(mClient, LOG_TAG, "The sixel repeat value " + repeat + " is greater than max repeat value " + TerminalSixel.SIXEL__MAX_REPEAT); + return false; + } + // Graphics Carriage Return `$`. // > The $ (2/4) character indicates the end of the sixel line. The active position returns // > to the left page border of the same sixel line. You can use this character to overprint lines. @@ -161,15 +163,11 @@ public boolean readData(int codePoint, int repeat) { } if (mBitmap.getWidth() < mCurX + repeat) { - int newBitmapWidth = mCurX + repeat + SIXEL__BITMAP_RESIZE_EXTRA; + int newBitmapWidth = mCurX + repeat + SIXEL__BITMAP_RESIZE_EXTRA_DIMENSION; if (newBitmapWidth < 0) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap width overflowed: " + mCurX + " (cursor x) + " + repeat + " (repeat) + " + SIXEL__BITMAP_RESIZE_EXTRA); - return false; - } - - if (newBitmapWidth > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap width " + newBitmapWidth + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap width overflowed: " + + mCurX + " (cursor x) + " + repeat + " (repeat) + " + SIXEL__BITMAP_RESIZE_EXTRA_DIMENSION); return false; } @@ -181,15 +179,11 @@ public boolean readData(int codePoint, int repeat) { if (mBitmap.getHeight() < mCurY + SIXEL__LINE_LEN) { // Very unlikely to resize both at the same time. - int newBitmapHeight = mCurY + SIXEL__BITMAP_RESIZE_EXTRA; + int newBitmapHeight = mCurY + SIXEL__BITMAP_RESIZE_EXTRA_DIMENSION; if (newBitmapHeight < 0) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap height overflowed: " + mCurY + " (cursor y) + " + SIXEL__BITMAP_RESIZE_EXTRA); - return false; - } - - if (newBitmapHeight > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap height " + newBitmapHeight + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); + Logger.logError(mClient, LOG_TAG, "The new sixel bitmap height overflowed: " + + mCurY + " (cursor y) + " + SIXEL__BITMAP_RESIZE_EXTRA_DIMENSION); return false; } @@ -251,20 +245,8 @@ public boolean resize(int sixelWidth, int sixelHeight) { int newBitmapHeight = Math.max(sixelHeight, bitmapHeight); if (bitmapWidth < newBitmapWidth || bitmapHeight < newBitmapHeight) { - if (newBitmapWidth > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap resize width " + newBitmapWidth + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); - return false; - } - - if (newBitmapHeight > TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION) { - Logger.logError(mClient, LOG_TAG, "The new sixel bitmap resize height " + newBitmapHeight + " is greater than max bitmap dimension " + TerminalSixel.SIXEL__MAX_BITMAP_DIMENSION); - return false; - } - mBitmap = TerminalBitmap.resizeBitmap(LOG_TAG, "sixel", mClient, mBitmap, newBitmapWidth, newBitmapHeight); - if (mBitmap == null) { - return false; - } + return mBitmap != null; } return true; From 253eee9ba5412e60911dd1c04697f9fdf7be86a6 Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Tue, 14 Apr 2026 16:05:44 +0500 Subject: [PATCH 10/15] Changed: Increase OCS 52 buffer limit to `100KB` for text to set to clipboard --- .../main/java/com/termux/terminal/TerminalEmulator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index aada13b78f..5c6dcdec7b 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -2694,6 +2694,14 @@ void setOscTypeVariables() { Integer terminalControlArgsCapacity = null; Integer terminalControlArgsMaxLength = null; switch (mOscType) { + case 52: + // Android has a `~100KB` limit for sharing/sending `UTF-16` encoded `String` + // with binder tranasactions, including clipboard, otherwise can result in a + // `TransactionTooLargeException`. + // - https://www.reddit.com/r/tasker/comments/prro8t/autoshare_crashed_when_i_pasted_the_file_path/ + terminalControlArgsMaxLength = (100 * 1024) + + /* `52;Pc;` */ 10; + break; case 1337: // iTerm image command sends the base64 encoded image, do not run complex logic for each byte. mIsFastPathOsc = true; mIgnoreCrLfForOsc = true; From 2151d08496b184980bdf89351213c7617a54e0d3 Mon Sep 17 00:00:00 2001 From: agnostic-apollo Date: Wed, 6 May 2026 02:33:12 +0500 Subject: [PATCH 11/15] Fixed: Allow sixel DCS command start `P3` arg to not end with a semicolon and allow not passing all optional parameters --- .../src/main/java/com/termux/terminal/TerminalEmulator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index 5c6dcdec7b..4595dbefd0 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -1400,6 +1400,10 @@ private int[] getSixelDcsSetupArgs(String dcs, int index) { } incArg = true; + } else if (ch == 'q') { + // If optional parameters `P1`, `P2` or `P3` are not all passed, or + // a parameter did not end with a `;`, but is followed by `q`. + return args; } else { break; } From 9add03d0a795a3c93d2694a63801c4515489c0e2 Mon Sep 17 00:00:00 2001 From: Kota Matsumoto Date: Wed, 5 Aug 2026 16:17:00 +0900 Subject: [PATCH 12/15] Fixed: Do not create a terminal bitmap for an image whose bitmap bounds could not be decoded as it hangs the terminal `TerminalBitmap.build()` first decodes only the bounds of the image data received for an iTerm image (`OSC 1337`) by calling `BitmapFactory.decodeByteArray()` with `BitmapFactory.Options.inJustDecodeBounds` set to `true`, so that an image that is too large to be drawn is not decoded at all. If `BitmapFactory` fails to decode the bounds, then `BitmapFactory.Options.outWidth` and `outHeight` are set to `0` instead of the image dimensions, which was not checked before the values were used. A `0` value for `imageWidth` or `imageHeight` results in a division by `0` while calculating the `wFactor` and `hFactor` scaling factors for an image whose aspect ratio is to be preserved, and in the `scaleFactor` loop after it never terminating, since the `imageHeight >= 2 * newHeight * scaleFactor` and `imageWidth >= 2 * newWidth * scaleFactor` conditions stay `true` for `0 >= 0` for any value of `scaleFactor`, until `scaleFactor` overflows to `0` after `201` iterations and then stays `0` forever. The terminal is hung on the thread that processes the terminal output and does not recover. The hang is triggered by sending an iTerm image command with a `width` parameter and image data that cannot be decoded, like `OSC 1337;File=inline=1;width=10: ST`. --- .../main/java/com/termux/terminal/TerminalBitmap.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java index 188f06bc2a..a4408d71eb 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java @@ -215,6 +215,17 @@ public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, imageHeight = options.outHeight; imageWidth = options.outWidth; + if (imageWidth < 1 || imageHeight < 1) { + // The image dimensions are not known if the image data could not be decoded. + // Bailing out early is required since a `0` dimension would otherwise result in + // a division by `0` while calculating the factors below and in the `scaleFactor` + // loop below never terminating. + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " from image byte array failed:" + + " Decoded bitmap bounds " + imageWidth + "x" + imageHeight + " are not valid"); + return null; + } + if (shouldPreserveAspectRatio) { double wFactor = 9999.0; double hFactor = 9999.0; From 52c2c8d9f0498da1a91929568f5aef8c06d83822 Mon Sep 17 00:00:00 2001 From: Kota Matsumoto Date: Wed, 5 Aug 2026 16:17:15 +0900 Subject: [PATCH 13/15] Fixed: Draw the pending text run before a terminal bitmap so that text before an image on the same line is not lost `TerminalRenderer.render()` accumulates consecutive columns of a line that have the same style into a text run, which is drawn with a single `drawTextRun()` call once the style changes or the end of the line is reached. The branch for a cell whose style is a terminal bitmap did not draw the run that was still pending before it and only reset the run state, so any text before an image on the same line was discarded and never drawn at all. The start of the run after the bitmap was also wrong. The `lastRunStartColumn` was set to `column + 1` after `column` had already been incremented past the cell of the bitmap, so the next run started one column too late and its text was drawn shifted by one cell. The `lastRunStartIndex` was set before `currentCharIndex` was incremented past the cell of the bitmap, so it pointed at the character of the bitmap cell instead of at the first character of the next run. Both issues affect sixel (`DCS q`) and iTerm (`OSC 1337`) images equally and are reproduced by printing text before an image escape sequence without a newline between them. --- .../java/com/termux/view/TerminalRenderer.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java index 787902ce08..a1619340e9 100644 --- a/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java +++ b/terminal-view/src/main/java/com/termux/view/TerminalRenderer.java @@ -104,6 +104,18 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, final int codePoint = charIsHighsurrogate ? Character.toCodePoint(charAtIndex, line[currentCharIndex + 1]) : charAtIndex; final long style = lineObject.getStyle(column); if (TextStyle.isTerminalBitmap(style)) { + // The text run that is pending must be drawn before the bitmap, otherwise the + // text before an image on the same line would never be drawn at all. + if (lastRunStartColumn >= 0 && column > lastRunStartColumn) { + final int columnWidthSinceLastRun = column - lastRunStartColumn; + final int charsSinceLastRun = currentCharIndex - lastRunStartIndex; + int cursorColor = lastRunInsideCursor ? mEmulator.mColors.mCurrentColors[TextStyle.COLOR_INDEX_CURSOR] : 0; + boolean invertCursorTextColor = lastRunInsideCursor && cursorShape == TerminalEmulator.TERMINAL_CURSOR_STYLE_BLOCK; + drawTextRun(canvas, line, palette, heightOffset, lastRunStartColumn, columnWidthSinceLastRun, + lastRunStartIndex, charsSinceLastRun, measuredWidthForRun, + cursorColor, cursorShape, lastRunStyle, reverseVideo || invertCursorTextColor || lastRunInsideSelection); + } + Bitmap bitmap = mEmulator.getScreen().getSixelBitmap(style); if (bitmap != null) { float left = column * mFontWidth; @@ -113,13 +125,15 @@ public final void render(TerminalEmulator mEmulator, Canvas canvas, int topRow, canvas.drawBitmap(bitmap, bitmapSrcRect, bitmapDestRect, null); } column += 1; + currentCharIndex += charsForCodePoint; + // The next text run starts after the cell of the bitmap, both in columns and in + // the characters of the line. measuredWidthForRun = 0.f; lastRunStyle = 0; lastRunInsideCursor = false; - lastRunStartColumn = column + 1; + lastRunStartColumn = column; lastRunStartIndex = currentCharIndex; lastRunFontWidthMismatch = false; - currentCharIndex += charsForCodePoint; continue; } final int codePointWcWidth = WcWidth.width(codePoint); From 13318246d9762094286b9fc16cbc7824be4c4adb Mon Sep 17 00:00:00 2001 From: Kota Matsumoto Date: Wed, 5 Aug 2026 16:17:44 +0900 Subject: [PATCH 14/15] Fixed: Leave the cursor after an image on the last row it covers if the column after it is the last column of the screen After a terminal bitmap has been added to the screen for an image, the cursor is moved to the column after the image on the last row the image covers if that position is still on the screen, and is otherwise wrapped to the first column of the row after the image. The condition used to check this was `col < mColumns - 1`, where `col` is the `0` based index of the column after the last column the image covers, which is off by one. The position is on the screen as long as `col` is a valid column index, so the condition must be `col < mColumns`. With the old condition, an image whose following column is the last column of the screen was treated as if it did not fit, and so one row more than the image covers was consumed and the cursor was left at the first column of that row. Note that this changes the behaviour for sixel (`DCS q`) and iTerm (`OSC 1337`) images, since this is the shared code path for the cursor position after any terminal bitmap. On a `56` column screen with the cursor at column `54`, printing an iTerm image that is `2` cells wide and `1` cell high covers the columns `54` and `55` and the cursor position reported by `CSI 6 n` changes from row `7` column `1` to row `6` column `56`. --- .../src/main/java/com/termux/terminal/TerminalEmulator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index 4595dbefd0..f232d69748 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -2975,7 +2975,7 @@ else if (controlCommandPrefix.equals("FileEnd")) { iTermImage.shouldPreserveAspectRatio()); int col = cursorDelta[1] + mCursorCol; - if (col < mColumns - 1) { + if (col < mColumns) { cursorDelta[0] -= 1; } else { col = 0; From 27eea1b963dce732e62d4f89ae0a9b71b86a0168 Mon Sep 17 00:00:00 2001 From: Kota Matsumoto Date: Wed, 5 Aug 2026 16:19:31 +0900 Subject: [PATCH 15/15] Added: Add terminal support for kitty graphics protocol images (`APC _G`) Support for displaying images sent with the kitty graphics protocol has been added on top of the existing terminal bitmap infrastructure that is used for sixel (`DCS q`) and iTerm (`OSC 1337`) images. A command is in the format `APC _ G [; ] ST`, where `control data` is a comma separated list of `=` pairs with single character keys and `payload` is the `base64` encoded image data. Only a minimal subset of the protocol is supported, which is enough to transmit an image and display it. The `CSI` `Primary Device Attributes` response is not changed, since clients detect kitty graphics protocol support by sending an `a=q` query command and checking the response for it. The following is supported: - The `a=t` (transmit), `a=T` (transmit and display), `a=p` (display an image that was already transmitted), `a=q` (query) and `a=d` (delete) actions. - The `t=d` (direct) transmission medium, where the image data is sent in the command payload. - The `f=100` (`PNG`), `f=24` (raw `RGB`) and `f=32` (raw `RGBA`) image formats, with the pixel dimensions of the raw formats passed with the `s` and `v` keys. - The `m=1`/`m=0` chunked transmission of the image data of a single image with multiple commands. - The `i=` and `p=` keys, and the `d=a`, `d=A`, `d=i` and `d=I` delete modes. - The `c=` and `r=` keys for the number of cells to display the image in. - The `x=`, `y=`, `w=` and `h=` keys for displaying only a source rectangle of the transmitted image. - The `C=1` key for not moving the cursor after displaying an image. - The `q=` key for suppressing success and error responses. The following is **not** supported and results in an `ENOTSUP` error response: - The `t=f`, `t=t` and `t=s` transmission mediums, where the image data is read from a file or shared memory. Reading arbitrary files that the terminal has access to on behalf of a client is a security concern, so this is intentionally not implemented. - The `o=z` (`zlib`) compressed image data. - The `a=f` (animation frame), `a=a` (animate) and `a=c` (compose) actions. The keys for unsupported features that a client is expected to be able to send for images this terminal can display are ignored instead of resulting in an error response. These are the `z` (z-index), `X` and `Y` (cell pixel offset) and `U` (unicode placeholder) keys. Not honouring `X` and `Y` means an image is aligned to the cell grid instead of being offset by up to one cell width and height. Implementation: - `KittyImage` parses the control data of a command, decodes the `base64` image data as it is received and holds the state of an image that is transmitted with multiple commands. The `base64` image data is not collected in `TerminalEmulator.mTerminalControlArgs` first, so a client may send an entire image with a single command instead of splitting it into the chunks of at most `4096` bytes the protocol recommends, and the total image data size is only bound by `KittyImage.IMAGE_DATA__MAX_LENGTH` (`TerminalBitmap.MAX_BITMAP_SIZE`). Only the control data is stored in the args buffer, which is limited to `4KB` by `TerminalEmulator.KITTY_GRAPHICS_CONTROL_DATA__MAX_LENGTH`, and a command with longer control data is discarded instead of being printed on the terminal as text. - `TerminalEmulator` gained an APC command type, set by `setApcTypeVariables()` from the first code point after the `APC` escape sequence, so that a kitty graphics command takes the fast path in `processCodePoint()` while all other APC commands keep being parsed and ignored like `xterm` does. The command is dispatched by `doApcKittyGraphics()` and responded to with `APC _Gi=;OK ST` or `APC _Gi=;: ST` as per the `q` key. - The image data transmitted with the `a=t` and `a=T` actions is stored for its image id in `TerminalEmulator.mKittyImages` so that it can be displayed again with an `a=p` command without being transmitted again. The map is in insertion order and the oldest images are evicted first once `KITTY_IMAGES__MAX_COUNT` (`32`) or `KITTY_IMAGES__MAX_TOTAL_SIZE` (`8MB`) is exceeded, and it is cleared on a terminal reset. - `TerminalBitmap.buildForKittyImage()` creates the bitmap for a placement. It cannot use the iTerm image path since the source rectangle of the `x=`, `y=`, `w=` and `h=` keys must be cropped before the image is scaled to the cells it is displayed in, otherwise the wrong part of the image would be scaled, and the raw `RGB`/`RGBA` formats are not decoded by `BitmapFactory`. The scaling calculation that both paths share has been extracted into `getScaledImageSize()`. A `PNG` is only subsampled while decoding if the entire image is displayed, since the source rectangle coordinates are in the coordinate space of the full size image, and its size is checked against `MAX_BITMAP_SIZE` before decoding instead. - `TerminalBuffer.addTerminalBitmapForKittyImage()` adds the bitmap to the screen, and `deleteKittyImagePlacements()` and `deleteAllKittyImagePlacements()` remove the bitmaps of the placements deleted by an `a=d` command and clear the cells of the screen and the transcript they were rendered in. The image id and placement id are stored in the `TerminalBitmap` itself instead of a separate registry so that they do not need to be kept in sync when bitmaps are removed and their bitmap numbers are reused. - `TerminalBuffer.releaseUnreferencedTerminalBitmaps()` releases the bitmaps of the cells that a new bitmap overwrote if they are no longer referenced by any cell. This does not rely on `doTerminalBitmapsGC()`, which only runs once per `timeDelta`, since an `a=p` command can display an already transmitted image with around 30 bytes, so a client that displays a new image at the same position for every frame would otherwise keep every one of them in memory. `KittyGraphicsTest` has been added with 74 tests covering the control data parsing, the supported and unsupported actions, keys and formats, the response and quiet levels, chunked transmission, the cursor position after an image, deletion and the image data limits. The comment of `ApcTest.testApcConsumed()` has been updated, since the `yazi` capability query it sends is now answered on stdin instead of being consumed silently. - https://sw.kovidgoyal.net/kitty/graphics-protocol/ --- .../java/com/termux/terminal/KittyImage.java | 991 +++++++++++++ .../com/termux/terminal/TerminalBitmap.java | 394 ++++- .../com/termux/terminal/TerminalBuffer.java | 159 +- .../com/termux/terminal/TerminalEmulator.java | 611 +++++++- .../java/com/termux/terminal/ApcTest.java | 6 +- .../termux/terminal/KittyGraphicsTest.java | 1315 +++++++++++++++++ 6 files changed, 3430 insertions(+), 46 deletions(-) create mode 100644 terminal-emulator/src/main/java/com/termux/terminal/KittyImage.java create mode 100644 terminal-emulator/src/test/java/com/termux/terminal/KittyGraphicsTest.java diff --git a/terminal-emulator/src/main/java/com/termux/terminal/KittyImage.java b/terminal-emulator/src/main/java/com/termux/terminal/KittyImage.java new file mode 100644 index 0000000000..38f3f111f6 --- /dev/null +++ b/terminal-emulator/src/main/java/com/termux/terminal/KittyImage.java @@ -0,0 +1,991 @@ +package com.termux.terminal; + +import java.util.Arrays; + +/** + * A kitty graphics protocol image received via an `APC _G` command. + * + * The command is in the format `APC _ G [; ] ST`, where `control data` is a + * comma separated list of `=` pairs with single character keys, and `payload` is the + * `base64` encoded image data. + * + * Only a minimal subset of the protocol is supported, check + * {@link TerminalEmulator#doApcKittyGraphics(KittyImage)} for details of what is supported and what + * is not. + * + * A single image may be transmitted with multiple commands by setting the `m` key to `1` for all + * commands except the last one, in which case the same {@link KittyImage} instance is kept by + * {@link TerminalEmulator} until the final command is received. + * + * The `base64` payload is not collected as text. Each character of it is decoded by + * {@link #readImageChar(char)} as it is received and the bytes are accumulated in + * {@link #mImageData}, since a `char` is stored on 2 bytes and `base64` requires 4 characters for + * every 3 bytes, so collecting the payload as text would require `2.66` times as much memory as the + * image data itself. The payload of a command does not have to be a multiple of the 4 characters + * that decode to 3 bytes, so the bits that do not complete a byte are carried over to the next + * command of the image in {@link #mBase64Buffer} and {@link #mBase64BufferBits}. The combined + * length of the image data of all the commands for a single image is bound by + * {@link #IMAGE_DATA__MAX_LENGTH}. + * + * - https://sw.kovidgoyal.net/kitty/graphics-protocol/ + */ +public class KittyImage { + + public static final String LOG_TAG = "KittyImage"; + + + + /** The {@link Enum} that defines {@link KittyImage} state. */ + public enum ImageState { + + INIT("init", 0), + ARGUMENTS_READ("arguments_read", 1), + IMAGE_READING("image_reading", 2), + IMAGE_READ("image_read", 3), + IMAGE_DECODED("image_decoded", 4), + FAILED("failed", 5); + + private final String name; + private final int value; + + ImageState(final String name, final int value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public int getValue() { + return value; + } + + } + + + + /* The supported actions for the `a` key. */ + + /** `a=t` - Transmit the image data and store it for the image id passed with the `i` key. */ + public static final char ACTION__TRANSMIT = 't'; + /** `a=T` - Like {@link #ACTION__TRANSMIT}, but also display the image at the cursor position. */ + public static final char ACTION__TRANSMIT_AND_DISPLAY = 'T'; + /** `a=p` - Display an image that was already transmitted for the image id passed with the `i` key. */ + public static final char ACTION__PUT = 'p'; + /** `a=q` - Query the terminal for kitty graphics protocol support without storing anything. */ + public static final char ACTION__QUERY = 'q'; + /** `a=d` - Delete image placements as per the {@link #mDeleteMode} passed with the `d` key. */ + public static final char ACTION__DELETE = 'd'; + + /** The default action if the `a` key is not passed. */ + public static final char ACTION__DEFAULT = ACTION__TRANSMIT; + + + /* The delete modes for the `d` key of an {@link #ACTION__DELETE} command. */ + + /** The `d` key was not passed, which is equivalent to `d=a`. */ + public static final char DELETE_MODE__NONE = 0; + /** `d=a` - Delete all placements. The uppercase `d=A` also frees the image data. */ + public static final char DELETE_MODE__ALL = 'a'; + public static final char DELETE_MODE__ALL_AND_FREE_DATA = 'A'; + /** + * `d=i` - Delete the placements of the image id passed with the `i` key, and only the placement + * for the placement id passed with the `p` key if it is passed. The uppercase `d=I` also frees + * the image data. + */ + public static final char DELETE_MODE__ID = 'i'; + public static final char DELETE_MODE__ID_AND_FREE_DATA = 'I'; + + + /* The transmission mediums for the `t` key. */ + + /** `t=d` - The image data is passed directly in the `base64` encoded command payload. */ + public static final char MEDIUM__DIRECT = 'd'; + + /** The default transmission medium if the `t` key is not passed. */ + public static final char MEDIUM__DEFAULT = MEDIUM__DIRECT; + + + /* The image formats for the `f` key. */ + + /** `f=24` - Raw `RGB` pixel data. */ + public static final int FORMAT__RGB = 24; + /** `f=32` - Raw `RGBA` pixel data. */ + public static final int FORMAT__RGBA = 32; + /** `f=100` - `PNG` image data. */ + public static final int FORMAT__PNG = 100; + + /** The default image format if the `f` key is not passed as per the protocol. */ + public static final int FORMAT__DEFAULT = FORMAT__RGBA; + + + /* The quiet levels for the `q` key. */ + + /** `q=0` - Send both success and error responses. */ + public static final int QUIET__NONE = 0; + /** `q=1` - Suppress success responses. */ + public static final int QUIET__SUCCESS = 1; + /** `q=2` - Suppress both success and error responses. */ + public static final int QUIET__ALL = 2; + + + /* The error codes sent in error responses. */ + + /** The control data of the command was malformed or out of range. */ + public static final String ERROR__EINVAL = "EINVAL"; + /** The command requires a feature of the protocol that is not supported. */ + public static final String ERROR__ENOTSUP = "ENOTSUP"; + /** No image data has been transmitted for the image id passed with the `i` key. */ + public static final String ERROR__ENOENT = "ENOENT"; + /** The `base64` encoded payload of the command could not be decoded. */ + public static final String ERROR__EBADDATA = "EBADDATA"; + /** The image data could not be decoded into a bitmap for display. */ + public static final String ERROR__EBADPNG = "EBADPNG"; + /** The image data exceeded the size limits of the terminal. */ + public static final String ERROR__ENOSPC = "ENOSPC"; + + + /** The image id value for the `i` key if it was not passed. Image id `0` is not valid. */ + public static final long IMAGE_ID__NONE = 0; + + /** The max value for the `i` key, since image ids are 32-bit unsigned integers. */ + public static final long IMAGE_ID__MAX = 4294967295L; + + /** The placement id value for the `p` key if it was not passed. Placement id `0` is not valid. */ + public static final long PLACEMENT_ID__NONE = 0; + + /** The max value for the `p` key, since placement ids are 32-bit unsigned integers. */ + public static final long PLACEMENT_ID__MAX = 4294967295L; + + /** + * The max value for the `c` and `r` keys. The x and y coordinates of a bitmap cell are stored + * in 12 bits each by {@link TextStyle#encodeTerminalBitmap(int, int, int)}. + */ + public static final int PLACEMENT_CELLS__MAX = 4095; + + /** + * The max value for the `s` and `v` keys, which is the max dimension of a bitmap supported by + * skia. Check {@link TerminalBitmap#MAX_BITMAP_SIZE} for more info. + */ + public static final int PIXEL_DIMENSION__MAX = 32766; + + /** + * The max length in bytes of the image data of a single image after it has been decoded from + * `base64`, which for the raw {@link #FORMAT__RGB} and {@link #FORMAT__RGBA} formats is + * ` * * `. + * + * The limit is derived from {@link TerminalBitmap#MAX_BITMAP_SIZE}, which is the max size of the + * pixels of a bitmap the terminal can actually draw, since accepting image data that is larger + * than that would only waste memory on an image that cannot be displayed, while accepting less + * would reject images that could have been displayed. + * + * A bitmap stores each pixel on 4 bytes as {@link android.graphics.Bitmap.Config#ARGB_8888}, so + * the max number of pixels is `MAX_BITMAP_SIZE / 4`, and the raw {@link #FORMAT__RGBA} format + * transmits the same 4 bytes per pixel, which makes the max image data length + * `(MAX_BITMAP_SIZE / 4) * 4`, so equal to `MAX_BITMAP_SIZE` itself. The raw + * {@link #FORMAT__RGB} format transmits only 3 bytes per pixel, so its data will be under this + * limit before its bitmap reaches `MAX_BITMAP_SIZE`, which is checked separately while creating + * the bitmap. The `PNG` format is compressed, so its data is far under this limit in practice. + */ + public static final int IMAGE_DATA__MAX_LENGTH = TerminalBitmap.MAX_BITMAP_SIZE; + + /** The initial capacity for {@link #mImageData}. */ + private static final int IMAGE_DATA__INITIAL_CAPACITY = 4096; + + + + private static final int BASE64__INVALID = -1; + + /** The max number of `=` padding characters a `base64` encoded string can end with. */ + private static final int BASE64__MAX_PADDING = 2; + + private static final int[] BASE64__DECODE_TABLE = createBase64DecodeTable(); + + + + protected final TerminalSessionClient mClient; + + /** The action passed with the `a` key. */ + protected char mAction = ACTION__DEFAULT; + /** Whether the `a` key was passed. A continuation chunk of an image does not pass it. */ + protected boolean mHasAction = false; + + /** The delete mode passed with the `d` key. */ + protected char mDeleteMode = DELETE_MODE__NONE; + + /** The transmission medium passed with the `t` key. */ + protected char mMedium = MEDIUM__DEFAULT; + + /** The image format passed with the `f` key. */ + protected int mFormat = FORMAT__DEFAULT; + + /** The image id passed with the `i` key. */ + protected long mImageId = IMAGE_ID__NONE; + + /** The placement id passed with the `p` key. */ + protected long mPlacementId = PLACEMENT_ID__NONE; + + /** Whether more chunks of the image will follow as per the `m` key. */ + protected boolean mMoreChunks = false; + + /** The quiet level passed with the `q` key. */ + protected int mQuiet = QUIET__NONE; + /** Whether the `q` key was passed. */ + protected boolean mHasQuiet = false; + + /** The number of columns and rows to display the image in as passed with the `c` and `r` keys. */ + protected int mColumns = -1; + protected int mRows = -1; + + /** + * The width and height in pixels of the image data as passed with the `s` and `v` keys, which + * are required for the raw {@link #FORMAT__RGB} and {@link #FORMAT__RGBA} formats. + */ + protected int mPixelWidth = -1; + protected int mPixelHeight = -1; + + /** + * The source rectangle of the transmitted image to display as passed with the `x`, `y`, `w` and + * `h` keys, where `x` and `y` are the top left corner in pixels and `w` and `h` are the size in + * pixels. The `x` and `y` keys default to `0` and the `w` and `h` keys default to the rest of + * the image after the offset, so `-1` means that they were not passed. + */ + protected int mSourceX = 0; + protected int mSourceY = 0; + protected int mSourceWidth = -1; + protected int mSourceHeight = -1; + + /** Whether the cursor must not be moved after displaying the image as per the `C` key. */ + protected boolean mDoNotMoveCursor = false; + + /** The error code and message to send in an error response if the command failed. */ + protected String mErrorCode; + protected String mErrorMessage; + + /** + * The image data decoded from the `base64` payload of all the chunks of the image received so + * far, which may be longer than {@link #mImageDataLength} since it is grown in advance. + * + * The payload of each chunk is decoded as it is received instead of the `base64` data of all the + * chunks being collected first and decoded at the end, since a `char` is stored on 2 bytes and + * `base64` requires 4 characters for every 3 bytes, so collecting it would require `2.66` times + * as much memory as the image data itself. + */ + protected byte[] mImageData; + + /** The number of bytes of {@link #mImageData} that have been decoded. */ + protected int mImageDataLength; + + /** + * The exact length in bytes the image data must have, which is only known in advance for the raw + * formats, otherwise `0`. It is used to allocate {@link #mImageData} exactly once and to reject + * a client that sends more data than it declared before the memory for it is allocated. + */ + protected int mImageDataExpectedLength; + + /** + * The bits of the `base64` payload that did not complete a byte yet, which are carried over to + * the next chunk of the image since the payload of a chunk does not have to be a multiple of the + * 4 characters that decode to 3 bytes. + */ + protected int mBase64Buffer; + protected int mBase64BufferBits; + + /** The number of `=` padding characters received at the end of the `base64` payload. */ + protected int mBase64Padding; + + /** The total number of `base64` payload characters received, including the padding. */ + protected int mBase64Length; + + /** The current state of the {@link ImageState}. */ + protected ImageState mCurrentState = ImageState.INIT; + /** The previous state of the {@link ImageState}. */ + protected ImageState mPreviousState = ImageState.INIT; + + + + protected KittyImage(TerminalSessionClient client) { + mClient = client; + } + + + + public char getAction() { + return mAction; + } + + public boolean hasAction() { + return mHasAction; + } + + + public char getDeleteMode() { + return mDeleteMode; + } + + + public int getFormat() { + return mFormat; + } + + /** Whether the image data is raw pixel data instead of an encoded image like `PNG`. */ + public boolean isRawFormat() { + return mFormat == FORMAT__RGB || mFormat == FORMAT__RGBA; + } + + /** Get the number of bytes each pixel of raw pixel data is stored on, or `0` for other formats. */ + public static int getBytesPerPixel(int format) { + switch (format) { + case FORMAT__RGB: return 3; + case FORMAT__RGBA: return 4; + default: return 0; + } + } + + + public long getImageId() { + return mImageId; + } + + public long getPlacementId() { + return mPlacementId; + } + + + public boolean hasMoreChunks() { + return mMoreChunks; + } + + + public int getQuiet() { + return mQuiet; + } + + + /** + * Get the width in pixels to display the image in, or `-1` if the `c` key was not passed, in + * which case the native width of the image is used. + */ + public int getWidthPixels(int cellWidthPixels) { + return mColumns > 0 ? mColumns * cellWidthPixels : -1; + } + + /** + * Get the height in pixels to display the image in, or `-1` if the `r` key was not passed, in + * which case the native height of the image is used. + */ + public int getHeightPixels(int cellHeightPixels) { + return mRows > 0 ? mRows * cellHeightPixels : -1; + } + + /** + * Whether the aspect ratio of the image should be preserved. If both the `c` and `r` keys are + * passed, then the image is scaled to exactly fill the requested cells as per the protocol. + */ + public boolean shouldPreserveAspectRatio() { + return !(mColumns > 0 && mRows > 0); + } + + + public int getPixelWidth() { + return mPixelWidth; + } + + public int getPixelHeight() { + return mPixelHeight; + } + + + public int getSourceX() { + return mSourceX; + } + + public int getSourceY() { + return mSourceY; + } + + public int getSourceWidth() { + return mSourceWidth; + } + + public int getSourceHeight() { + return mSourceHeight; + } + + /** + * Validate the source rectangle passed with the `x`, `y`, `w` and `h` keys against the pixel + * dimensions of the image it is for, so that a client that requests a rectangle outside the + * image is sent an error response instead of the request being silently clipped. + * + * The dimensions of an image are only known before it is decoded into a bitmap if the client + * passed the `s` and `v` keys when transmitting it, which is always required for the raw formats + * but is optional for `PNG`, so the source rectangle is validated against the actual dimensions + * of the bitmap again by + * {@link TerminalBitmap#buildForKittyImage(TerminalBuffer, int, int, byte[], int, int, int, int, int, int, int, int, int, int, int, int, boolean)}. + * + * @param imageWidth The width in pixels of the image, or a value `< 1` if it is not known. + * @param imageHeight The height in pixels of the image, or a value `< 1` if it is not known. + * @return Returns `true` if the source rectangle is valid, otherwise `false`, in which case + * {@link #getErrorCode()} will be set. + */ + public synchronized boolean validateSourceRectangle(int imageWidth, int imageHeight) { + if (imageWidth < 1 || imageHeight < 1) return true; + + if (mSourceX >= imageWidth) { + return setStateFailed(ERROR__EINVAL, "source rectangle x " + mSourceX + + " is outside image width " + imageWidth); + } + + if (mSourceWidth > 0 && mSourceX + mSourceWidth > imageWidth) { + return setStateFailed(ERROR__EINVAL, "source rectangle x " + mSourceX + + " and width " + mSourceWidth + " exceed image width " + imageWidth); + } + + if (mSourceY >= imageHeight) { + return setStateFailed(ERROR__EINVAL, "source rectangle y " + mSourceY + + " is outside image height " + imageHeight); + } + + if (mSourceHeight > 0 && mSourceY + mSourceHeight > imageHeight) { + return setStateFailed(ERROR__EINVAL, "source rectangle y " + mSourceY + + " and height " + mSourceHeight + " exceed image height " + imageHeight); + } + + return true; + } + + + /** Whether the cursor should be moved to after the image after displaying it. */ + public boolean shouldMoveCursor() { + return !mDoNotMoveCursor; + } + + + public String getErrorCode() { + return mErrorCode; + } + + public String getErrorMessage() { + return mErrorMessage; + } + + public boolean isFailed() { + return mErrorCode != null; + } + + + /** Get the decoded image data, which is only valid after {@link #finishImage()} succeeded. */ + public byte[] getDecodedImage() { + return mImageData; + } + + + protected synchronized boolean setState(ImageState newState) { + // The `ImageState.FAILED` state can always be set, since a command can fail after its image + // data was decoded successfully, like if it requests a source rectangle that is outside the + // image or if the bitmap for it cannot be created. + // Any other state transition cannot go back or change if already at `ImageState.IMAGE_DECODED` + if (newState != ImageState.FAILED && + (newState.getValue() < mCurrentState.getValue() || mCurrentState == ImageState.IMAGE_DECODED)) { + Logger.logError(mClient, LOG_TAG, + "Invalid image state transition from \"" + mCurrentState.getName() + "\" to " + "\"" + newState.getName() + "\""); + return false; + } + + // The `ImageState.FAILED` can be set again, like to add more errors, but we don't update + // `mPreviousState` with the `mCurrentState` value if its at `ImageState.FAILED` to + // preserve the last valid state. + if (mCurrentState != ImageState.FAILED) + mPreviousState = mCurrentState; + + mCurrentState = newState; + return true; + } + + + /** + * Set {@link ImageState#FAILED} state with the error code and message to send in the error + * response for the command. + * + * @return Returns `false` always so that callers can return the value directly. + */ + protected synchronized boolean setStateFailed(String errorCode, String errorMessage) { + // Only keep the first error so that the error response is for the actual cause of failure. + if (mErrorCode == null) { + mErrorCode = errorCode; + mErrorMessage = errorMessage; + } + + Logger.logError(mClient, LOG_TAG, "Kitty graphics command failed with " + errorCode + ": " + errorMessage); + setState(ImageState.FAILED); + return false; + } + + + protected synchronized boolean ensureState(ImageState expectedState, String functionName) { + if (mCurrentState != expectedState) { + Logger.logError(mClient, LOG_TAG, + "The current image state is \"" + mCurrentState.getName() + "\" but expected \"" + expectedState.getName() + "\"" + + " while calling '" + functionName + "'"); + return false; + } + return true; + } + + + /** + * Read the `` of an `APC _G [; ] ST` command. + * + * All the control data is read even if a key is invalid, so that the `i` and `q` keys required + * to send the response for the command are read regardless of the order the keys are passed in. + * + * @param apcArgs The `APC` command arguments received, without the leading `ESC _`. + * @param index The index in `apcArgs` at which the control data starts. + * @return Returns the index at which the `base64` encoded payload starts, which will be equal + * to the length of `apcArgs` if the command has no payload, or `-1` if the control data was + * invalid, in which case {@link #getErrorCode()} will be set. + */ + public synchronized int readControlData(StringBuilder apcArgs, int index) { + if (!ensureState(ImageState.INIT, "KittyImage.readControlData()")) { + return -1; + } + + int argsLength = apcArgs.length(); + int payloadIndex = argsLength; + + while (index < argsLength) { + // End of control data, the rest of the command is the payload. + if (apcArgs.charAt(index) == ';') { + payloadIndex = index + 1; + break; + } + + // All control data keys are a single character followed by a `=`. + char argKey = apcArgs.charAt(index); + if (index + 1 >= argsLength || apcArgs.charAt(index + 1) != '=') { + setStateFailed(ERROR__EINVAL, "malformed control data"); + + // Skip the malformed key so that the remaining keys are still read. + while (index < argsLength && apcArgs.charAt(index) != ',' && apcArgs.charAt(index) != ';') { + index++; + } + if (index < argsLength && apcArgs.charAt(index) == ',') index++; + continue; + } + + int valueStartIndex = index + 2; + int valueEndIndex = valueStartIndex; + while (valueEndIndex < argsLength) { + char ch = apcArgs.charAt(valueEndIndex); + if (ch == ',' || ch == ';') break; + valueEndIndex++; + } + + readControlDataArg(argKey, apcArgs.substring(valueStartIndex, valueEndIndex)); + + index = valueEndIndex; + // Skip the `,` separator, the `;` is handled at the start of the loop. + if (index < argsLength && apcArgs.charAt(index) == ',') index++; + } + + if (isFailed()) { + return -1; + } + + setState(ImageState.ARGUMENTS_READ); + + return payloadIndex; + } + + /** Read a single `=` pair of the control data. */ + private synchronized boolean readControlDataArg(char argKey, String argValue) { + switch (argKey) { + case 'a': // The action. + if (argValue.length() != 1) return setStateFailed(ERROR__EINVAL, "invalid action"); + mAction = argValue.charAt(0); + mHasAction = true; + switch (mAction) { + case ACTION__TRANSMIT: + case ACTION__TRANSMIT_AND_DISPLAY: + case ACTION__PUT: + case ACTION__QUERY: + case ACTION__DELETE: + break; + default: + // The `a=f` (animation frame), `a=a` (animate) and `a=c` (compose) actions + // are valid protocol actions, but are not supported. + return setStateFailed(ERROR__ENOTSUP, "unsupported action"); + } + break; + case 'd': // The delete mode. + if (argValue.length() != 1) return setStateFailed(ERROR__EINVAL, "invalid delete mode"); + mDeleteMode = argValue.charAt(0); + break; + case 't': // The transmission medium. + if (argValue.length() != 1) return setStateFailed(ERROR__EINVAL, "invalid transmission medium"); + mMedium = argValue.charAt(0); + if (mMedium != MEDIUM__DIRECT) { + // Reading the image data from a file, a temporary file or shared memory is not + // supported. Reading arbitrary files that the terminal has access to on behalf + // of a client is a security concern. + return setStateFailed(ERROR__ENOTSUP, "unsupported transmission medium"); + } + break; + case 'o': // The compression. + if (argValue.length() != 1 || argValue.charAt(0) != 'z') { + return setStateFailed(ERROR__EINVAL, "invalid compression"); + } + return setStateFailed(ERROR__ENOTSUP, "unsupported compression"); + case 'f': { // The image format. + long value = readNumberArg(argValue, 0, FORMAT__PNG); + if (value != FORMAT__RGB && value != FORMAT__RGBA && value != FORMAT__PNG) { + return setStateFailed(ERROR__EINVAL, "invalid image format"); + } + mFormat = (int) value; + break; + } + case 'i': { // The image id. + long value = readNumberArg(argValue, 1, IMAGE_ID__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid image id"); + mImageId = value; + break; + } + case 'p': { // The placement id. + long value = readNumberArg(argValue, 1, PLACEMENT_ID__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid placement id"); + mPlacementId = value; + break; + } + case 'm': { // Whether more chunks will follow. + long value = readNumberArg(argValue, 0, 1); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid chunk flag"); + mMoreChunks = value == 1; + break; + } + case 'q': { // The quiet level. + long value = readNumberArg(argValue, 0, QUIET__ALL); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid quiet level"); + mQuiet = (int) value; + mHasQuiet = true; + break; + } + case 'c': { // The number of columns to display the image in. + long value = readNumberArg(argValue, 0, PLACEMENT_CELLS__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid columns"); + mColumns = (int) value; + break; + } + case 'r': { // The number of rows to display the image in. + long value = readNumberArg(argValue, 0, PLACEMENT_CELLS__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid rows"); + mRows = (int) value; + break; + } + case 's': { // The width in pixels of the raw pixel data. + long value = readNumberArg(argValue, 0, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid pixel width"); + mPixelWidth = (int) value; + break; + } + case 'v': { // The height in pixels of the raw pixel data. + long value = readNumberArg(argValue, 0, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid pixel height"); + mPixelHeight = (int) value; + break; + } + case 'C': { // Whether to not move the cursor after displaying the image. + long value = readNumberArg(argValue, 0, 1); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid cursor movement policy"); + mDoNotMoveCursor = value == 1; + break; + } + case 'x': { // The x coordinate in pixels of the source rectangle. + long value = readNumberArg(argValue, 0, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid source rectangle x"); + mSourceX = (int) value; + break; + } + case 'y': { // The y coordinate in pixels of the source rectangle. + long value = readNumberArg(argValue, 0, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid source rectangle y"); + mSourceY = (int) value; + break; + } + case 'w': { // The width in pixels of the source rectangle. + long value = readNumberArg(argValue, 1, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid source rectangle width"); + mSourceWidth = (int) value; + break; + } + case 'h': { // The height in pixels of the source rectangle. + long value = readNumberArg(argValue, 1, PIXEL_DIMENSION__MAX); + if (value < 0) return setStateFailed(ERROR__EINVAL, "invalid source rectangle height"); + mSourceHeight = (int) value; + break; + } + default: + // The keys for unsupported features of the protocol are read but ignored, since a + // client is expected to pass them for images that can still be displayed. These are + // the `I` (image number), `z` (z-index) and `U` (unicode placeholder) keys, the `S`, + // `O`, `P` and `Q` keys for the transmission mediums and placement features that are + // not supported, and any key that is not part of the protocol at all. + // The `X` and `Y` keys, which offset the image by pixels inside the first cell it is + // displayed in, are also ignored, so an image will be aligned to the cell grid + // instead of being offset by up to one cell width and height. + // The values of the ignored keys are not validated since they are not used. + break; + } + return true; + } + + /** + * Read an unsigned decimal control data value. + * + * @return Returns the value read, or `-1` if it was not a decimal number or was not within the + * `minValue` and `maxValue` range. + */ + private static long readNumberArg(String argValue, long minValue, long maxValue) { + // A value of more than 10 digits cannot be a valid 32-bit unsigned integer, and the check + // also prevents an overflow of `value` below. + if (argValue.isEmpty() || argValue.length() > 10) return -1; + + long value = 0; + for (int i = 0; i < argValue.length(); i++) { + char ch = argValue.charAt(i); + if (ch < '0' || ch > '9') return -1; + value = (value * 10) + (ch - '0'); + } + + return (value < minValue || value > maxValue) ? -1 : value; + } + + + /** + * Read the control data of a continuation chunk of the image being received by this instance. + * As per the protocol, a continuation chunk only passes the `m` and `q` keys, and all the other + * keys are inherited from the first chunk of the image. + * + * @param continuationChunk The {@link KittyImage} whose control data was read for the + * continuation chunk command. + */ + public synchronized void readContinuationControlData(KittyImage continuationChunk) { + mMoreChunks = continuationChunk.hasMoreChunks(); + + // The quiet level of the first chunk of the image is kept if the continuation chunk did not + // pass the `q` key, so that a response is not sent for an image whose transmission was + // started with responses suppressed. + if (continuationChunk.mHasQuiet) { + mQuiet = continuationChunk.getQuiet(); + mHasQuiet = true; + } + } + + + /** + * Start receiving the `base64` encoded `` of an `APC _G ; ST` + * command, which is decoded by {@link #readImageChar(char)} as it is received instead of being + * collected first, so that the size of the payload of a single command is not limited. + * + * @return Returns `true` if the payload can be received, otherwise `false`, in which case + * {@link #getErrorCode()} will be set. + */ + public synchronized boolean startImage() { + if (mCurrentState != ImageState.IMAGE_READING && + !ensureState(ImageState.ARGUMENTS_READ, "KittyImage.startImage()")) { + return false; + } + + // The exact length of the image data is known in advance for the raw formats, so the limit + // can be checked before all of it has been received, and the buffer for it can be allocated + // exactly once with the required size instead of being grown as more chunks are received. + if (isRawFormat() && mImageData == null && mPixelWidth > 0 && mPixelHeight > 0) { + long imageDataLength = (long) mPixelWidth * mPixelHeight * getBytesPerPixel(mFormat); + if (imageDataLength > IMAGE_DATA__MAX_LENGTH) { + return setStateFailed(ERROR__ENOSPC, "image data exceeds max length " + IMAGE_DATA__MAX_LENGTH); + } + + mImageDataExpectedLength = (int) imageDataLength; + if (!ensureImageDataCapacity(mImageDataExpectedLength)) return false; + } + + // An empty payload can be received as well, so the state is changed even if no character of + // the payload is received at all. + setState(ImageState.IMAGE_READING); + + return true; + } + + /** + * Decode a single `base64` character of the payload of the current chunk of the image. + * + * The payload of a chunk does not have to be a multiple of the 4 `base64` characters that decode + * to 3 bytes, so the bits that do not complete a byte are carried over to the next chunk. + * + * @return Returns `true` if the character was decoded, otherwise `false`, in which case + * {@link #getErrorCode()} will be set. The remaining characters of the payload must still be + * passed to this method so that they are not printed on the terminal as text, and they will be + * discarded. + */ + public synchronized boolean readImageChar(char ch) { + // The rest of the payload is discarded once the command has failed. + if (isFailed()) return false; + + if (ch == '=') { + // The padding is only valid at the very end of the `base64` data, so it can only be + // received in the final chunk of the image. + if (mMoreChunks) { + return setStateFailed(ERROR__EBADDATA, "base64 padding in a chunk that is not the final chunk"); + } + + mBase64Padding++; + mBase64Length++; + return true; + } + + if (mBase64Padding > 0) { + return setStateFailed(ERROR__EBADDATA, "base64 data after the padding"); + } + + int value = (ch < 128) ? BASE64__DECODE_TABLE[ch] : BASE64__INVALID; + if (value < 0) { + return setStateFailed(ERROR__EBADDATA, "invalid base64 image data"); + } + + mBase64Length++; + mBase64Buffer = (mBase64Buffer << 6) | value; + mBase64BufferBits += 6; + + if (mBase64BufferBits >= 8) { + mBase64BufferBits -= 8; + + // A client that sends more data than the dimensions it declared is rejected before the + // memory for the extra data is allocated. + if (mImageDataExpectedLength > 0 && mImageDataLength >= mImageDataExpectedLength) { + return setStateFailed(ERROR__EINVAL, "image data exceeds the " + mImageDataExpectedLength + + " bytes required for pixel dimensions " + mPixelWidth + "x" + mPixelHeight); + } + + if (mImageData == null || mImageDataLength >= mImageData.length) { + if (!ensureImageDataCapacity(mImageDataLength + 1)) return false; + } + + mImageData[mImageDataLength++] = (byte) (mBase64Buffer >>> mBase64BufferBits); + } + + return true; + } + + /** + * Ensure {@link #mImageData} can hold `capacity` bytes. + * + * @return Returns `true` if the buffer can hold `capacity` bytes, otherwise `false`, in which + * case {@link #getErrorCode()} will be set. + */ + private synchronized boolean ensureImageDataCapacity(int capacity) { + if (capacity > IMAGE_DATA__MAX_LENGTH) { + return setStateFailed(ERROR__ENOSPC, "image data exceeds max length " + IMAGE_DATA__MAX_LENGTH); + } + + if (mImageData != null && mImageData.length >= capacity) return true; + + // The buffer is grown by doubling its size so that it is not reallocated and copied for + // every chunk received, since the total length of the image data of a compressed image + // format is not known in advance. + long newCapacity = Math.max(capacity, IMAGE_DATA__INITIAL_CAPACITY); + if (mImageData != null) { + newCapacity = Math.max(newCapacity, Math.min((long) mImageData.length * 2, IMAGE_DATA__MAX_LENGTH)); + } + + try { + // Allocating the new array and copying the old one into it can cause an OOM. + mImageData = (mImageData == null) ? new byte[(int) newCapacity] : Arrays.copyOf(mImageData, (int) newCapacity); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + return setStateFailed(ERROR__ENOSPC, "allocating " + newCapacity + " bytes for image data failed: " + t.getMessage()); + } + + return true; + } + + + /** + * Finish decoding the image data received for all the chunks of the image and validate it + * against the image format and dimensions passed in the control data. + * + * @return Returns `true` if the image data is valid, otherwise `false`, in which case + * {@link #getErrorCode()} will be set. + */ + public synchronized boolean finishImage() { + if (!ensureState(ImageState.IMAGE_READING, "KittyImage.finishImage()")) { + return setStateFailed(ERROR__EBADDATA, "incomplete image data"); + } + + setState(ImageState.IMAGE_READ); + + // A single remaining `base64` character only holds 6 bits and cannot encode a byte. + if (mBase64BufferBits == 6) { + return setStateFailed(ERROR__EBADDATA, "invalid base64 image data"); + } + + // The padding does not add any bits, so it is not required to decode the data, but if it is + // present, then there can be at most two padding characters and the total number of + // characters must be a multiple of four. + if (mBase64Padding > 0 && (mBase64Padding > BASE64__MAX_PADDING || (mBase64Length % 4) != 0)) { + return setStateFailed(ERROR__EBADDATA, "invalid base64 image data"); + } + + if (mImageDataLength < 1) { + mImageData = null; + return setStateFailed(ERROR__EBADDATA, "empty image data"); + } + + // The pixel dimensions of raw image data are not part of the data itself, so they must be + // passed with the `s` and `v` keys and must match the length of the data exactly. + if (isRawFormat()) { + if (mPixelWidth < 1 || mPixelHeight < 1) { + int imageDataLength = mImageDataLength; + mImageData = null; + return setStateFailed(ERROR__EINVAL, "missing pixel dimensions for" + + " image data of length " + imageDataLength); + } + + if (mImageDataLength != mImageDataExpectedLength) { + int imageDataLength = mImageDataLength; + mImageData = null; + return setStateFailed(ERROR__EINVAL, "image data of length " + imageDataLength + + " does not match the " + mImageDataExpectedLength + " bytes required for" + + " pixel dimensions " + mPixelWidth + "x" + mPixelHeight); + } + } + + // The buffer is grown in advance, so the extra bytes at its end must be released. This never + // creates a copy of the image data of a raw format image, since its buffer was allocated with + // the exact length required. + if (mImageData.length != mImageDataLength) { + try { + mImageData = Arrays.copyOf(mImageData, mImageDataLength); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + return setStateFailed(ERROR__ENOSPC, "trimming image data to" + + " length " + mImageDataLength + " failed: " + t.getMessage()); + } + } + + setState(ImageState.IMAGE_DECODED); + return true; + } + + + private static int[] createBase64DecodeTable() { + int[] table = new int[128]; + Arrays.fill(table, BASE64__INVALID); + + String alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + for (int i = 0; i < alphabet.length(); i++) { + table[alphabet.charAt(i)] = i; + } + return table; + } + +} diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java index a4408d71eb..35748f3d69 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBitmap.java @@ -10,7 +10,9 @@ import android.graphics.RectF; import android.os.Build; +import java.util.HashSet; import java.util.Properties; +import java.util.Set; /** * A terminal bitmap for images. @@ -138,6 +140,25 @@ private static int initMaxBitmapSize() { protected int[] mCursorDelta; + /** + * Whether this bitmap is a placement of a kitty graphics image, in which case + * {@link #mKittyImageId} and {@link #mKittyPlacementId} define which placement it is of which + * image, although both may be unset if the client did not pass the `i` and `p` keys. + * + * The association is stored in the {@link TerminalBitmap} itself instead of a separate registry + * so that it does not need to be kept in sync when bitmaps are removed by + * {@link TerminalBuffer#doTerminalBitmapsGC(int)} and + * {@link TerminalBuffer#removeScrolledOutTerminalBitmaps(int)}, since bitmap numbers are reused + * for new bitmaps. + */ + protected boolean mIsKittyImage; + + /** The kitty graphics image id this bitmap is a placement of, if {@link #mIsKittyImage}. */ + protected long mKittyImageId = KittyImage.IMAGE_ID__NONE; + + /** The kitty graphics placement id of this bitmap, if {@link #mIsKittyImage}. */ + protected long mKittyPlacementId = KittyImage.PLACEMENT_ID__NONE; + protected TerminalBitmap(TerminalSessionClient client, int bitmapNum, Bitmap bitmap, int cellWidth, int cellHeight, @@ -226,26 +247,9 @@ public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, return null; } - if (shouldPreserveAspectRatio) { - double wFactor = 9999.0; - double hFactor = 9999.0; - if (width > 0) { - wFactor = (double) width / imageWidth; - } - if (height > 0) { - hFactor = (double) height / imageHeight; - } - double factor = Math.min(wFactor, hFactor); - newWidth = (int) (factor * imageWidth); - newHeight = (int) (factor * imageHeight); - } else { - if (height <= 0) { - newHeight = imageHeight; - } - if (width <= 0) { - newWidth = imageWidth; - } - } + int[] newSize = getScaledImageSize(imageWidth, imageHeight, width, height, shouldPreserveAspectRatio); + newWidth = newSize[0]; + newHeight = newSize[1]; int scaleFactor = 1; while (imageHeight >= 2 * newHeight * scaleFactor && imageWidth >= 2 * newWidth * scaleFactor) { @@ -348,6 +352,319 @@ public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, } + /** + * Build a {@link TerminalBitmap} for a placement of a kitty graphics image. + * + * The kitty graphics protocol has its own image formats and allows a source rectangle of the + * image to be displayed instead of the entire image, so + * {@link #build(TerminalBuffer, int, byte[], int, int, int, int, int, int, boolean)} used for + * iTerm images cannot be used. The source rectangle must be cropped before the image is scaled + * to the cells it is to be displayed in, otherwise the wrong part of the image would be scaled. + * + * Note that unlike the iTerm image path, a `PNG` is decoded at its full size instead of being + * subsampled while decoding, since the source rectangle coordinates are in the coordinate space + * of the full size image. The size of the bitmap that would be created is checked against + * {@link #MAX_BITMAP_SIZE} before decoding instead. + * + * @param format The kitty graphics image format, check {@link KittyImage#FORMAT__PNG} and + * {@link KittyImage#isRawFormat()} for more info. + * @param image The image data, which is the raw pixel data for the raw formats, in which case + * its length must be `pixelWidth * pixelHeight * `. + * @param pixelWidth The width in pixels of the image data, required for the raw formats. + * @param pixelHeight The height in pixels of the image data, required for the raw formats. + * @param sourceX The x coordinate in pixels of the source rectangle of the image to display. + * @param sourceY The y coordinate in pixels of the source rectangle of the image to display. + * @param sourceWidth The width in pixels of the source rectangle, or a value `< 1` for the rest + * of the image after `sourceX`. + * @param sourceHeight The height in pixels of the source rectangle, or a value `< 1` for the + * rest of the image after `sourceY`. + */ + public static TerminalBitmap buildForKittyImage(TerminalBuffer terminalBuffer, int bitmapNum, + int format, byte[] image, + int pixelWidth, int pixelHeight, + int sourceX, int sourceY, + int sourceWidth, int sourceHeight, + int x, int y, int cellWidth, int cellHeight, + int width, int height, boolean shouldPreserveAspectRatio) { + try { + if (image == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed: Image data not set"); + return null; + } + + // A `PNG` can only be subsampled while decoding it if the entire image is displayed, + // since the source rectangle coordinates are in the coordinate space of the full size + // image, so subsampling would crop the wrong region of a smaller image. + boolean hasSourceRectangle = sourceX > 0 || sourceY > 0 || sourceWidth > 0 || sourceHeight > 0; + + Bitmap newBitmap; + if (format == KittyImage.FORMAT__PNG) { + newBitmap = createBitmapFromPng(terminalBuffer.getClient(), bitmapNum, image, + !hasSourceRectangle, width, height, shouldPreserveAspectRatio); + } else { + newBitmap = createBitmapFromRawPixels(terminalBuffer.getClient(), bitmapNum, image, + KittyImage.getBytesPerPixel(format), pixelWidth, pixelHeight); + } + + if (newBitmap == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed: New bitmap not set"); + return null; + } + + newBitmap = cropBitmapToSourceRectangle(terminalBuffer.getClient(), bitmapNum, newBitmap, + sourceX, sourceY, sourceWidth, sourceHeight); + if (newBitmap == null) { + return null; + } + + int bitmapWidth = newBitmap.getWidth(); + int bitmapHeight = newBitmap.getHeight(); + if (bitmapWidth < 1 || bitmapHeight < 1) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed:" + + " The bitmap dimensions " + bitmapWidth + "x" + bitmapHeight + " are not valid"); + return null; + } + + if (height > 0 || width > 0) { + int[] newSize = getScaledImageSize(bitmapWidth, bitmapHeight, width, height, shouldPreserveAspectRatio); + if (newSize[0] < 1 || newSize[1] < 1) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed:" + + " The scaled size " + newSize[0] + "x" + newSize[1] + " is not valid"); + return null; + } + + if (newSize[0] != bitmapWidth || newSize[1] != bitmapHeight) { + try { + newBitmap = Bitmap.createScaledBitmap(newBitmap, newSize[0], newSize[1], true); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed:" + + " Create scaled bitmap failed: " + t.getMessage()); + return null; + } + + if (newBitmap == null) { + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed: Scaled bitmap not set"); + return null; + } + } + } + + newBitmap = resizeBitmapConstrained(LOG_TAG, "kitty image", terminalBuffer.getClient(), newBitmap, + newBitmap.getWidth(), newBitmap.getHeight(), cellWidth, cellHeight, + terminalBuffer.mColumns - x); + TerminalBitmap terminalBitmap = build(terminalBuffer, bitmapNum, newBitmap, x, y, cellWidth, cellHeight); + if (terminalBitmap == null) { + return terminalBitmap; + } + + terminalBitmap.setCursorDelta(new int[] { + terminalBitmap.getScrollLines(), + (terminalBitmap.getBitmap().getWidth() + cellWidth - 1) / cellWidth}); + + return terminalBitmap; + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(terminalBuffer.getClient(), LOG_TAG, + "Create terminal bitmap " + bitmapNum + " for kitty image failed: " + t.getMessage()); + return null; + } + } + + /** + * Create a {@link Bitmap} from `PNG` image data of a kitty graphics image. + * + * @param canSubsample Whether the image may be subsampled while decoding it to save memory, + * which requires the entire image to be displayed, check + * {@link #buildForKittyImage(TerminalBuffer, int, int, byte[], int, int, int, int, int, int, int, int, int, int, int, int, boolean)} + * for more info. + * @param width The width in pixels the image is to be displayed in, or a value `< 1` if it is to + * be displayed at its own size, in which case it cannot be subsampled. + * @param height The height in pixels the image is to be displayed in, or a value `< 1` if it is + * to be displayed at its own size, in which case it cannot be subsampled. + */ + private static Bitmap createBitmapFromPng(TerminalSessionClient client, int bitmapNum, byte[] image, + boolean canSubsample, int width, int height, + boolean shouldPreserveAspectRatio) { + try { + // Get image dimensions without creating a bitmap so that an image that is too large to + // be drawn is not decoded at all. + BitmapFactory.Options options = new BitmapFactory.Options(); + options.inJustDecodeBounds = true; + BitmapFactory.decodeByteArray(image, 0, image.length, options); + + int imageWidth = options.outWidth; + int imageHeight = options.outHeight; + + if (imageWidth > 0 && imageHeight > 0) { + if (!isBitmapSizeValid(client, bitmapNum, imageWidth, imageHeight)) { + return null; + } + + if (canSubsample && (width > 0 || height > 0)) { + int[] newSize = getScaledImageSize(imageWidth, imageHeight, width, height, shouldPreserveAspectRatio); + if (newSize[0] > 0 && newSize[1] > 0) { + int scaleFactor = 1; + while (imageHeight >= 2 * newSize[1] * scaleFactor && imageWidth >= 2 * newSize[0] * scaleFactor) { + scaleFactor = scaleFactor * 2; + } + + if (scaleFactor > 1) { + // Subsample the original image to get a smaller image to save memory. + BitmapFactory.Options scaleOptions = new BitmapFactory.Options(); + scaleOptions.inSampleSize = scaleFactor; + return BitmapFactory.decodeByteArray(image, 0, image.length, scaleOptions); + } + } + } + } + + return BitmapFactory.decodeByteArray(image, 0, image.length); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(client, LOG_TAG, "Create bitmap for" + + " terminal bitmap " + bitmapNum + " from png image data failed: " + t.getMessage()); + return null; + } + } + + /** + * Crop a {@link Bitmap} to the source rectangle of a kitty graphics placement. + * + * @return Returns the cropped bitmap, the original bitmap if the source rectangle covers the + * entire bitmap, or `null` if the source rectangle is not within the bounds of the bitmap. + */ + private static Bitmap cropBitmapToSourceRectangle(TerminalSessionClient client, int bitmapNum, Bitmap bitmap, + int sourceX, int sourceY, int sourceWidth, int sourceHeight) { + int bitmapWidth = bitmap.getWidth(); + int bitmapHeight = bitmap.getHeight(); + + // The width and height default to the rest of the image after the x and y offset. + int width = (sourceWidth > 0) ? sourceWidth : bitmapWidth - sourceX; + int height = (sourceHeight > 0) ? sourceHeight : bitmapHeight - sourceY; + + if (sourceX == 0 && sourceY == 0 && width == bitmapWidth && height == bitmapHeight) { + return bitmap; + } + + if (sourceX < 0 || sourceY < 0 || width < 1 || height < 1 || + sourceX + width > bitmapWidth || sourceY + height > bitmapHeight) { + Logger.logError(client, LOG_TAG, "Crop bitmap for" + + " terminal bitmap " + bitmapNum + " failed: The source rectangle" + + " " + sourceX + "," + sourceY + " " + width + "x" + height + + " is not within the bitmap dimensions " + bitmapWidth + "x" + bitmapHeight); + return null; + } + + try { + return Bitmap.createBitmap(bitmap, sourceX, sourceY, width, height); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(client, LOG_TAG, "Crop bitmap for" + + " terminal bitmap " + bitmapNum + " to source rectangle" + + " " + sourceX + "," + sourceY + " " + width + "x" + height + " failed: " + t.getMessage()); + return null; + } + } + + /** Whether a bitmap of the dimensions can be drawn as per {@link #MAX_BITMAP_SIZE}. */ + private static boolean isBitmapSizeValid(TerminalSessionClient client, int bitmapNum, int bitmapWidth, int bitmapHeight) { + long bitmapSize = (long) bitmapWidth * bitmapHeight * 4; + if (bitmapSize > MAX_BITMAP_SIZE) { + Logger.logError(client, LOG_TAG, + "The bitmap for terminal bitmap " + bitmapNum + " with" + + " dimensions " + bitmapWidth + "x" + bitmapHeight + + " has size " + bitmapSize + " greater than max bitmap size " + MAX_BITMAP_SIZE); + return false; + } + return true; + } + + /** + * Create an {@link Bitmap.Config#ARGB_8888} {@link Bitmap} from raw `RGB` or `RGBA` pixel data + * of a kitty graphics image. The alpha channel is expected to not be premultiplied. + */ + private static Bitmap createBitmapFromRawPixels(TerminalSessionClient client, int bitmapNum, byte[] pixels, + int bytesPerPixel, int pixelWidth, int pixelHeight) { + if (bytesPerPixel < 3 || pixelWidth < 1 || pixelHeight < 1 || + pixels.length != pixelWidth * pixelHeight * bytesPerPixel) { + Logger.logError(client, LOG_TAG, "Create bitmap for" + + " terminal bitmap " + bitmapNum + " from raw pixels failed:" + + " Pixel data of length " + pixels.length + + " does not match dimensions " + pixelWidth + "x" + pixelHeight + + " with " + bytesPerPixel + " bytes per pixel"); + return null; + } + + if (!isBitmapSizeValid(client, bitmapNum, pixelWidth, pixelHeight)) { + return null; + } + + try { + int[] colors = new int[pixelWidth * pixelHeight]; + for (int i = 0; i < colors.length; i++) { + int offset = i * bytesPerPixel; + int red = pixels[offset] & 0xff; + int green = pixels[offset + 1] & 0xff; + int blue = pixels[offset + 2] & 0xff; + int alpha = (bytesPerPixel > 3) ? (pixels[offset + 3] & 0xff) : 0xff; + colors[i] = (alpha << 24) | (red << 16) | (green << 8) | blue; + } + + return Bitmap.createBitmap(colors, pixelWidth, pixelHeight, Bitmap.Config.ARGB_8888); + } catch (Throwable t) { + if (t instanceof OutOfMemoryError) System.gc(); + Logger.logError(client, LOG_TAG, "Create bitmap for" + + " terminal bitmap " + bitmapNum + " from raw pixels with" + + " dimensions " + pixelWidth + "x" + pixelHeight + " failed: " + t.getMessage()); + return null; + } + } + + /** + * Get the size to scale an image to so that it is displayed in the requested `width` and + * `height` in pixels, where a value `<= 0` means that the respective dimension of the image + * itself should be used. + * + * @return Returns an array with the new width as the first value and the new height as the + * second value. + */ + private static int[] getScaledImageSize(int imageWidth, int imageHeight, + int width, int height, boolean shouldPreserveAspectRatio) { + int newWidth = width; + int newHeight = height; + + if (shouldPreserveAspectRatio) { + double wFactor = 9999.0; + double hFactor = 9999.0; + if (width > 0) { + wFactor = (double) width / imageWidth; + } + if (height > 0) { + hFactor = (double) height / imageHeight; + } + double factor = Math.min(wFactor, hFactor); + newWidth = (int) (factor * imageWidth); + newHeight = (int) (factor * imageHeight); + } else { + if (height <= 0) { + newHeight = imageHeight; + } + if (width <= 0) { + newWidth = imageWidth; + } + } + + return new int[] {newWidth, newHeight}; + } + + /** Build a {@link TerminalBitmap} from a {@link Bitmap}. */ public static TerminalBitmap build(TerminalBuffer terminalBuffer, int bitmapNum, Bitmap bitmap, int x, int y, int cellWidth, int cellHeight) { @@ -374,16 +691,31 @@ public static TerminalBitmap buildOrThrow(TerminalBuffer terminalBuffer, int bit int height = (bitmapHeight + cellHeight - 1) / cellHeight; int s = 0; + // The bitmaps of the cells that are overwritten by this bitmap may no longer be referenced by + // any cell afterwards, in which case they must be released, otherwise a client that displays + // a new image at the same position for every frame would keep all of them in memory. + Set overwrittenBitmapNums = null; + for (int i = 0; i < height; i++) { if (y + i - s == terminalBuffer.mScreenRows) { terminalBuffer.scrollDownOneLine(0, terminalBuffer.mScreenRows, TextStyle.NORMAL); s++; } for (int j = 0; j < width ; j++) { - terminalBuffer.setChar(x + j, y + i - s, '+', TextStyle.encodeTerminalBitmap(bitmapNum, j, i)); + int row = y + i - s; + + int overwrittenBitmapNum = TextStyle.getTerminalBitmapNum(terminalBuffer.getStyleAt(row, x + j)); + if (overwrittenBitmapNum >= TerminalBuffer.TERMINAL_BITMAP__NUM_START && overwrittenBitmapNum != bitmapNum) { + if (overwrittenBitmapNums == null) overwrittenBitmapNums = new HashSet<>(); + overwrittenBitmapNums.add(overwrittenBitmapNum); + } + + terminalBuffer.setChar(x + j, row, '+', TextStyle.encodeTerminalBitmap(bitmapNum, j, i)); } } + terminalBuffer.releaseUnreferencedTerminalBitmaps(overwrittenBitmapNums); + if (width * cellWidth < bitmapWidth) { bitmap = Bitmap.createBitmap(bitmap, 0, 0, width * cellWidth, bitmapHeight); } @@ -433,6 +765,26 @@ public void setCursorDelta(int[] cursorDelta) { } + public boolean isKittyImage() { + return mIsKittyImage; + } + + public long getKittyImageId() { + return mKittyImageId; + } + + public long getKittyPlacementId() { + return mKittyPlacementId; + } + + /** Mark this bitmap as a placement of a kitty graphics image. */ + public void setKittyImage(long kittyImageId, long kittyPlacementId) { + mIsKittyImage = true; + mKittyImageId = kittyImageId; + mKittyPlacementId = kittyPlacementId; + } + + diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java index 26ce7120c5..52e56704dc 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalBuffer.java @@ -2,6 +2,7 @@ import java.util.Arrays; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.HashMap; @@ -619,7 +620,7 @@ public synchronized int sixelEnd(int x, int y, int cellW, int cellH) { if (terminalBitmap == null || terminalBitmap.getBitmap() == null) { return 0; } - mTerminalBitmaps.put(bitmapNum, terminalBitmap); + addTerminalBitmap(terminalBitmap); doTerminalBitmapsGC(30000); return terminalBitmap.mScrollLines; @@ -700,13 +701,131 @@ public synchronized int[] addTerminalBitmapForImage(byte[] image, int x, int y, if (terminalBitmap == null || terminalBitmap.getBitmap() == null) { return new int[] {0, 0}; } - mTerminalBitmaps.put(bitmapNum, terminalBitmap); + addTerminalBitmap(terminalBitmap); doTerminalBitmapsGC(30000); return terminalBitmap.mCursorDelta; } + /** + * Add a {@link TerminalBitmap} to the screen for a placement of a kitty graphics image. + * + * @param format The kitty graphics image format, check {@link KittyImage#FORMAT__PNG} and + * {@link KittyImage#isRawFormat()} for more info. + * @param image The image data, which is the raw pixel data for the raw formats. + * @param pixelWidth The width in pixels of the image data, only required for the raw formats. + * @param pixelHeight The height in pixels of the image data, only required for the raw formats. + * @param sourceX The x coordinate in pixels of the source rectangle of the image to display. + * @param sourceY The y coordinate in pixels of the source rectangle of the image to display. + * @param sourceWidth The width in pixels of the source rectangle, or a value `< 1` for the rest + * of the image after `sourceX`. + * @param sourceHeight The height in pixels of the source rectangle, or a value `< 1` for the + * rest of the image after `sourceY`. + * @param kittyImageId The kitty graphics image id the bitmap is a placement of. + * @param kittyPlacementId The kitty graphics placement id of the bitmap. + * @return Returns an array with the number of rows the bitmap covers on the screen as the first + * value and the number of columns as the second value, or `{0, 0}` if the bitmap could not be + * created. + */ + public synchronized int[] addTerminalBitmapForKittyImage(int format, byte[] image, + int pixelWidth, int pixelHeight, + int sourceX, int sourceY, + int sourceWidth, int sourceHeight, + int x, int y, int cellW, int cellH, + int width, int height, boolean shouldPreserveAspectRatio, + long kittyImageId, long kittyPlacementId) { + int bitmapNum = getFreeTerminalBitmapNum(); + if (bitmapNum < TERMINAL_BITMAP__NUM_START) { + Logger.logError(mClient, LOG_TAG, "Cannot create more than " + TERMINAL_BITMAP__NUM_END + " bitmaps"); + return new int[] {0, 0}; + } + + TerminalBitmap terminalBitmap = TerminalBitmap.buildForKittyImage(this, bitmapNum, format, image, + pixelWidth, pixelHeight, sourceX, sourceY, sourceWidth, sourceHeight, x, y, + cellW, cellH, width, height, shouldPreserveAspectRatio); + + if (terminalBitmap == null || terminalBitmap.getBitmap() == null) { + return new int[] {0, 0}; + } + terminalBitmap.setKittyImage(kittyImageId, kittyPlacementId); + addTerminalBitmap(terminalBitmap); + + doTerminalBitmapsGC(30000); + return terminalBitmap.mCursorDelta; + } + + /** Add a {@link TerminalBitmap} to {@link #mTerminalBitmaps} for the bitmap number it was created with. */ + synchronized void addTerminalBitmap(TerminalBitmap terminalBitmap) { + mTerminalBitmaps.put(terminalBitmap.getBitmapNum(), terminalBitmap); + } + + + /** + * Delete the placements of a kitty graphics image and remove the {@link TerminalBitmap} that + * were created for them. + * + * @param kittyImageId The image id whose placements to delete. + * @param kittyPlacementId The placement id to delete, or {@link KittyImage#PLACEMENT_ID__NONE} + * to delete all the placements of the image. + */ + public synchronized void deleteKittyImagePlacements(long kittyImageId, long kittyPlacementId) { + deleteKittyImagePlacements(kittyImageId, kittyPlacementId, false); + } + + /** + * Delete the placements of all the kitty graphics images and remove the {@link TerminalBitmap} + * that were created for them. The bitmaps for sixel and iTerm images are not removed. + */ + public synchronized void deleteAllKittyImagePlacements() { + deleteKittyImagePlacements(KittyImage.IMAGE_ID__NONE, KittyImage.PLACEMENT_ID__NONE, true); + } + + private synchronized void deleteKittyImagePlacements(long kittyImageId, long kittyPlacementId, boolean allImages) { + if (mTerminalBitmaps.isEmpty()) return; + + Set bitmapsToRemove = new HashSet<>(); + for (Map.Entry terminalBitmap : mTerminalBitmaps.entrySet()) { + TerminalBitmap bitmap = terminalBitmap.getValue(); + if (!bitmap.isKittyImage()) continue; + if (!allImages && + (bitmap.getKittyImageId() != kittyImageId || + (kittyPlacementId != KittyImage.PLACEMENT_ID__NONE && + bitmap.getKittyPlacementId() != kittyPlacementId))) { + continue; + } + bitmapsToRemove.add(terminalBitmap.getKey()); + } + + if (bitmapsToRemove.isEmpty()) return; + + // Clear the cells of the screen and the transcript that the placements were rendered in. + for (TerminalRow line : mLines) { + if (line == null || !line.mHasTerminalBitmap) continue; + + boolean hasTerminalBitmap = false; + for (int column = 0; column < mColumns; column++) { + int bitmapNum = TextStyle.getTerminalBitmapNum(line.getStyle(column)); + if (bitmapNum < TERMINAL_BITMAP__NUM_START) continue; + + if (bitmapsToRemove.contains(bitmapNum)) { + line.setChar(column, ' ', TextStyle.NORMAL); + } else { + hasTerminalBitmap = true; + } + } + + // The flag is used to skip lines while removing and garbage collecting bitmaps, so it + // must be unset if the line no longer contains any bitmap cell. + line.mHasTerminalBitmap = hasTerminalBitmap; + } + + for (Integer bitmapNum : bitmapsToRemove) { + mTerminalBitmaps.remove(bitmapNum); + } + } + + /** Remove bitmaps that are completely scrolled out. */ public synchronized void removeScrolledOutTerminalBitmaps(int row) { Set bitmapsToRemove = new HashSet<>(); @@ -737,6 +856,42 @@ public synchronized void removeScrolledOutTerminalBitmaps(int row) { } } + /** + * Remove the {@link TerminalBitmap} for the bitmap numbers that are no longer referenced by any + * cell of the screen or the transcript, which is called by + * {@link TerminalBitmap#buildOrThrow(TerminalBuffer, int, Bitmap, int, int, int, int)} with the + * bitmap numbers of the cells that were overwritten by a new bitmap. + * + * This does not rely on {@link #doTerminalBitmapsGC(int)}, which only runs once per `timeDelta` + * and would otherwise let a client that displays a new image at the same position for every frame + * keep every one of them in memory, since the kitty graphics `a=p` action can display an image + * that was already transmitted with a command of only around 30 bytes. + * + * The bitmaps are not recycled, since they may still be referenced by the renderer of the + * terminal view on another thread, so only the reference held by this buffer is released and the + * memory is freed once the garbage collector runs. + * + * @param bitmapNums The bitmap numbers to release if they are unreferenced. The set is modified. + */ + synchronized void releaseUnreferencedTerminalBitmaps(Set bitmapNums) { + if (bitmapNums == null || bitmapNums.isEmpty()) return; + + for (TerminalRow line : mLines) { + if (line == null || !line.mHasTerminalBitmap) continue; + for (int column = 0; column < mColumns; column++) { + int bitmapNum = TextStyle.getTerminalBitmapNum(line.getStyle(column)); + if (bitmapNum >= TERMINAL_BITMAP__NUM_START && bitmapNums.remove(bitmapNum) && bitmapNums.isEmpty()) { + // All the bitmaps are still referenced by a cell. + return; + } + } + } + + for (Integer bitmapNum : bitmapNums) { + mTerminalBitmaps.remove(bitmapNum); + } + } + public synchronized void doTerminalBitmapsGC(int timeDelta) { if (mTerminalBitmaps.isEmpty() || mTerminalBitmapsLastGC + timeDelta > SystemClock.uptimeMillis()) { return; diff --git a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java index f232d69748..5735d34907 100644 --- a/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java +++ b/terminal-emulator/src/main/java/com/termux/terminal/TerminalEmulator.java @@ -4,7 +4,10 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Stack; @@ -160,7 +163,9 @@ public final class TerminalEmulator { * {@link #mTerminalControlArgs}, and for later code points {@link #receiveApc(int)} is called * instead. *

    - * Currently, APC commands are only parsed, but ignored as none are supported. + * The only APC commands supported are the kitty graphics protocol `APC _G` commands, check + * {@link #doApcKittyGraphics(KittyImage)} for more info. All other APC commands are parsed, but + * ignored. *

    * - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Application-Program-Command-functions */ @@ -353,6 +358,101 @@ public final class TerminalEmulator { + /** The {@link #ESC_APC} command type is not known yet. */ + private static final int APC_TYPE__NONE = -1; + /** The {@link #ESC_APC} command is not supported and its arguments will be ignored. */ + private static final int APC_TYPE__UNSUPPORTED = 0; + /** The {@link #ESC_APC} command is a kitty graphics protocol `APC _G` command. */ + private static final int APC_TYPE__KITTY_GRAPHICS = 1; + /** + * The {@link #ESC_APC} command cannot be processed, like if its control data is too long, so the + * rest of it is discarded instead of being printed on the terminal as text. + */ + private static final int APC_TYPE__DISCARD = 2; + + /** + * The Application Program Command `type` received as `ESC _ type`. + * This will be set as soon as the first code point after the `APC` escape sequence is received. + */ + private int mApcType = APC_TYPE__NONE; + + /** + * The initial capacity for the control data of a kitty graphics command stored in + * {@link #mTerminalControlArgs}. The `base64` encoded image data of a command is not stored in + * it, so only the `key=value` pairs before the `;` need to fit, which is normally under 64 + * characters. + */ + private static final int KITTY_GRAPHICS_CONTROL_DATA__INITIAL_CAPACITY = 128; + + /** + * The max length of the control data of a kitty graphics command stored in + * {@link #mTerminalControlArgs}, which is the `key=value` pairs before the `;` that starts the + * `base64` encoded image data. The protocol defines around 25 keys with values of at most 10 + * digits, so `4KB` is far more than a client can have a use for, and a command with control data + * longer than that is discarded instead of being printed on the terminal as text. + * + * The image data itself is not stored in {@link #mTerminalControlArgs} but is decoded as it is + * received by {@link KittyImage#readImageChar(char)}, so the size of the image data a single + * command can send is not limited, and is only bound in total by + * {@link KittyImage#IMAGE_DATA__MAX_LENGTH}. + */ + private static final int KITTY_GRAPHICS_CONTROL_DATA__MAX_LENGTH = 4 * 1024; + + /** + * The {@link KittyImage} if a kitty graphics image is being received with multiple commands, + * which is the case if the `m=1` key was passed with the last command received for the image. + */ + private KittyImage mKittyImage; + + /** + * The {@link KittyImage} of the kitty graphics command whose `base64` encoded image data is + * currently being received, which is set as soon as the `;` that ends the control data of the + * command has been received, and unset when the command has been processed. + */ + private KittyImage mKittyGraphicsPayload; + + /** + * The image data transmitted for a kitty graphics image id, which is kept so that the image can + * be displayed again with an `a=p` command without transmitting it again. + */ + private static final class KittyStoredImage { + + /** The image format the image data was transmitted with, check {@link KittyImage#getFormat()}. */ + final int mFormat; + + /** The image data, which is the raw pixel data for {@link KittyImage#isRawFormat()} formats. */ + final byte[] mImage; + + /** The width and height in pixels of the image data, only set for the raw formats. */ + final int mPixelWidth, mPixelHeight; + + KittyStoredImage(int format, byte[] image, int pixelWidth, int pixelHeight) { + mFormat = format; + mImage = image; + mPixelWidth = pixelWidth; + mPixelHeight = pixelHeight; + } + + } + + /** + * The images transmitted for kitty graphics image ids with the `a=t` and `a=T` actions. + * The map is in insertion order so that the oldest images can be evicted first if + * {@link #KITTY_IMAGES__MAX_COUNT} or {@link #KITTY_IMAGES__MAX_TOTAL_SIZE} is exceeded. + */ + private final LinkedHashMap mKittyImages = new LinkedHashMap<>(); + + /** The combined size in bytes of all the image data stored in {@link #mKittyImages}. */ + private int mKittyImagesTotalSize; + + /** The max number of images whose data is stored in {@link #mKittyImages}. */ + private static final int KITTY_IMAGES__MAX_COUNT = 32; + + /** The max combined size in bytes of all the image data stored in {@link #mKittyImages}. */ + private static final int KITTY_IMAGES__MAX_TOTAL_SIZE = 8 * 1024 * 1024; + + + /** * True if the current escape sequence should continue, false if the current escape sequence should be terminated. * Used when parsing a single character. @@ -2091,6 +2191,7 @@ private void doEsc(int b) { break; case '_': // APC - Application Program Command. clearTerminalControlArgs(); + clearApcTypeVariables(); continueSequence(ESC_APC); break; default: @@ -2594,7 +2695,41 @@ private void receiveApc(final int b) { continueSequence(ESC_APC__ESC); break; default: - if (!collectTerminalControlArgs(b)) return; + // The `base64` encoded image data of a kitty graphics command is decoded as it is + // received instead of being collected in the args buffer first, so that a client + // that sends an entire image with a single command is supported. + if (mKittyGraphicsPayload != null) { + mKittyGraphicsPayload.readImageChar((char) b); + continueSequence(ESC_APC); + return; + } + + if (mApcType == APC_TYPE__DISCARD) { + continueSequence(ESC_APC); + return; + } + + if (!collectTerminalControlArgs(b)) { + abortApc(); + return; + } + + if (mApcType == APC_TYPE__NONE) { + setApcTypeVariables(); + } + + if (mApcType == APC_TYPE__KITTY_GRAPHICS) { + if (b == ';') { + startApcKittyGraphicsPayload(); + } else if (mTerminalControlArgs.length() >= KITTY_GRAPHICS_CONTROL_DATA__MAX_LENGTH) { + Logger.logError(mClient, LOG_TAG, "Discarding kitty graphics command with" + + " control data longer than max length " + KITTY_GRAPHICS_CONTROL_DATA__MAX_LENGTH); + mKittyImage = null; + mApcType = APC_TYPE__DISCARD; + clearTerminalControlArgs(); + } + } + break; } } @@ -2604,28 +2739,470 @@ private void receiveApc(final int b) { private void receiveApcEsc(final int b) { switch (b) { case '\\': - //doApc(); - //clearApcTypeVariables(); + doApc(); + clearApcTypeVariables(); break; default: - // The ESC character was not followed by a \, so insert the ESC and - // the current character in arg buffer. - if (!collectTerminalControlArgs(27)) return; - if (!collectTerminalControlArgs(b)) return; + // The ESC character was not followed by a \, so it and the current character are + // part of the command data. + if (mKittyGraphicsPayload != null) { + mKittyGraphicsPayload.readImageChar((char) 27); + mKittyGraphicsPayload.readImageChar((char) b); + continueSequence(ESC_APC); + break; + } + + if (mApcType == APC_TYPE__DISCARD) { + continueSequence(ESC_APC); + break; + } + + // Insert the ESC and the current character in arg buffer. + if (!collectTerminalControlArgs(27)) { + abortApc(); + return; + } + if (!collectTerminalControlArgs(b)) { + abortApc(); + return; + } continueSequence(ESC_APC); break; } } + /** + * Abort an {@link #ESC_APC} command whose args could not be collected, like if the args buffer + * overflowed. The command will not be processed, and the data received for any kitty graphics + * image being transmitted with multiple commands is incomplete and must be dropped. + */ + private void abortApc() { + mKittyImage = null; + clearApcTypeVariables(); + } + + /** + * Start receiving the `base64` encoded image data of a kitty graphics command, which is called + * as soon as the `;` that ends its control data has been received, so that the image data can be + * decoded as it is received instead of being collected in {@link #mTerminalControlArgs} first. + */ + private void startApcKittyGraphicsPayload() { + KittyImage kittyImage = resolveApcKittyGraphicsCommand(); + + // The control data has been read, and the image data is not stored in the args buffer, so it + // is no longer needed for this command. + clearTerminalControlArgs(); + + if (!kittyImage.isFailed()) { + kittyImage.startImage(); + } + + // The image data is still passed to the command if it has already failed, so that it is + // discarded instead of being printed on the terminal as text. + mKittyGraphicsPayload = kittyImage; + } + + /** + * Read the control data of a kitty graphics command from {@link #mTerminalControlArgs} and get + * the {@link KittyImage} to process the command with, which is the {@link #mKittyImage} that is + * still being received if the command is a continuation chunk for it. + * + * @return Returns the {@link KittyImage} for the command, which will have + * {@link KittyImage#getErrorCode()} set if the control data was not valid. + */ + private KittyImage resolveApcKittyGraphicsCommand() { + KittyImage kittyImage = new KittyImage(mClient); + + // The `mTerminalControlArgs` contains `G[;]`. + if (kittyImage.readControlData(mTerminalControlArgs, /* `G` */ 1) < 0) { + // The control data is not valid, so an image that is still being received cannot be + // continued by this command either. + mKittyImage = null; + return kittyImage; + } + + // As per the protocol, a continuation chunk of an image being received with multiple + // commands only passes the `m` and `q` keys, so a command that passes the `a` key is always + // a new command, even if the previous image was not received completely. + if (mKittyImage != null) { + if (kittyImage.hasAction()) { + Logger.logWarn(mClient, LOG_TAG, "A new kitty graphics command received while an image was still being received"); + mKittyImage = null; + } else { + mKittyImage.readContinuationControlData(kittyImage); + kittyImage = mKittyImage; + } + } + + return kittyImage; + } + + /** + * Set {@link #ESC_APC} type variables. The type of an `APC` command is defined by the first + * code point received after the `APC` escape sequence. + */ + void setApcTypeVariables() { + if (mApcType != APC_TYPE__NONE) return; + if (mTerminalControlArgs.length() < 1) return; + + if (mTerminalControlArgs.charAt(0) == 'G') { + // Only the control data of a kitty graphics command is stored in the args buffer, since + // its image data is decoded as it is received, so only a small capacity is required. + mApcType = APC_TYPE__KITTY_GRAPHICS; + ensureTerminalControlArgsCapacity(KITTY_GRAPHICS_CONTROL_DATA__INITIAL_CAPACITY); + } else { + mApcType = APC_TYPE__UNSUPPORTED; + } + } + /** * Clear {@link #ESC_APC} type variables. + * + * The {@link #mKittyImage} is not cleared as sequential APC commands will be received for an + * image transmitted with multiple commands, and the variable is required to be set until the + * final command for it is received. */ - public void clearApcTypeVariables() {} + public void clearApcTypeVariables() { + mApcType = APC_TYPE__NONE; + mKittyGraphicsPayload = null; + } /** * Do {@link #ESC_APC}. Check its docs for more info. */ - private void doApc() {} + private void doApc() { + if (mKittyGraphicsPayload != null) { + // The image data of the command has already been received and decoded. + doApcKittyGraphics(mKittyGraphicsPayload); + } else if (mApcType == APC_TYPE__KITTY_GRAPHICS) { + // A command without any image data, like `a=p` and `a=d`. + KittyImage kittyImage = resolveApcKittyGraphicsCommand(); + if (!kittyImage.isFailed()) { + kittyImage.startImage(); + } + doApcKittyGraphics(kittyImage); + } else { + // All other APC commands are silently ignored, like xterm does. + if (LOG_ESCAPE_SEQUENCES) + Logger.logWarn(mClient, LOG_TAG, "Ignoring unsupported APC command"); + } + + // Free image data from memory held in apc command arguments as it is no longer needed. + clearTerminalControlArgs(); + + finishSequence(); + } + + + + /** + * Do a kitty graphics protocol `APC _ G [; ] ST` command. + * + * Only a minimal subset of the protocol is supported. + * + * The following is supported: + * - `a=t` (transmit), `a=T` (transmit and display), `a=p` (display an image that was already + * transmitted), `a=q` (query) and `a=d` (delete) actions. + * - `t=d` (direct) transmission medium, where the image data is sent in the command payload. + * - `f=100` (`PNG`), `f=24` (raw `RGB`) and `f=32` (raw `RGBA`) image formats, with the pixel + * dimensions of the raw formats passed with the `s` and `v` keys. + * - `m=1`/`m=0` chunked transmission of the image data with multiple commands. + * - `i=` and `p=`, and the `d=a`, `d=A`, `d=i` and `d=I` delete modes. + * - `c=`/`r=` for the number of cells to display the image in. + * - `x=`, `y=`, `w=` and `h=` for displaying only a source rectangle of the transmitted image, + * which is cropped out of the image before it is scaled to the cells it is displayed in. + * - `C=1` for not moving the cursor after displaying an image. + * - `q=` for suppressing success and error responses. + * + * The following is **not** supported and will result in an `ENOTSUP` error response: + * - `t=f`, `t=t` and `t=s` transmission mediums, where the image data is read from a file or + * shared memory. Reading arbitrary files that the terminal has access to on behalf of a + * client is a security concern, so this is intentionally not implemented. + * - `o=z` (`zlib`) compressed image data. + * - The `a=f` (animation frame), `a=a` (animate) and `a=c` (compose) actions. + * + * The image data of a command is decoded as it is received by {@link KittyImage#readImageChar(char)} + * instead of being collected in {@link #mTerminalControlArgs} first, so a client may send an + * entire image with a single command instead of splitting it into the chunks of at most `4096` + * bytes the protocol recommends. + * + * The keys for unsupported features are read but ignored instead of resulting in an error + * response, since a client is expected to be able to send them for images this terminal can + * display. Every key other than the `a`, `d`, `t`, `o`, `f`, `i`, `p`, `m`, `q`, `c`, `r`, `s`, + * `v`, `C`, `x`, `y`, `w` and `h` keys listed above is ignored, and its value is not validated + * since it is not used. These are the `I` (image number), `z` (z-index), `X` and `Y` (pixel + * offset of the image inside its first cell) and `U` (unicode placeholder) keys, the `S`, `O`, + * `P` and `Q` keys for the transmission mediums and placement features that are not supported, + * and any key that is not part of the protocol at all. Note that not honouring `X` and `Y` means + * an image is aligned to the cell grid instead of being offset by up to one cell width and + * height, and that not honouring `I` means a client must pass an image id with the `i` key for + * an image it wants to display again with an `a=p` command. + * + * - https://sw.kovidgoyal.net/kitty/graphics-protocol/ + */ + private void doApcKittyGraphics(KittyImage kittyImage) { + if (kittyImage.isFailed()) { + mKittyImage = null; + respondToKittyGraphicsCommand(kittyImage); + return; + } + + // Wait for the remaining chunks of the image before processing the command. No response is + // sent until the final chunk has been received. + if (kittyImage.hasMoreChunks()) { + mKittyImage = kittyImage; + return; + } + + mKittyImage = null; + + switch (kittyImage.getAction()) { + case KittyImage.ACTION__QUERY: + doKittyGraphicsQuery(kittyImage); + break; + case KittyImage.ACTION__TRANSMIT: + case KittyImage.ACTION__TRANSMIT_AND_DISPLAY: + doKittyGraphicsTransmit(kittyImage); + break; + case KittyImage.ACTION__PUT: + doKittyGraphicsPut(kittyImage); + break; + case KittyImage.ACTION__DELETE: + doKittyGraphicsDelete(kittyImage); + break; + default: + kittyImage.setStateFailed(KittyImage.ERROR__ENOTSUP, "unsupported action"); + break; + } + + respondToKittyGraphicsCommand(kittyImage); + } + + /** + * Do a kitty graphics `a=q` command. + * + * The command is used by clients to detect whether the terminal supports the protocol, and the + * image data passed with it must be validated but must not be stored or displayed. + */ + private void doKittyGraphicsQuery(KittyImage kittyImage) { + // The transmission medium, image format and compression have already been validated while + // reading the control data, so only the image data itself is left to validate. The image + // data of the `f=100` (`PNG`) format is not decoded into a bitmap for a query, since that + // would be an expensive operation for data that is going to be discarded. + kittyImage.finishImage(); + } + + /** Do a kitty graphics `a=t` or `a=T` command. */ + private void doKittyGraphicsTransmit(KittyImage kittyImage) { + if (!kittyImage.finishImage()) return; + + long imageId = kittyImage.getImageId(); + KittyStoredImage storedImage = new KittyStoredImage(kittyImage.getFormat(), + kittyImage.getDecodedImage(), kittyImage.getPixelWidth(), kittyImage.getPixelHeight()); + + // The image is only stored if it can be referenced by an image id with an `a=p` command later. + if (imageId != KittyImage.IMAGE_ID__NONE) { + storeKittyImage(imageId, storedImage); + } + + if (kittyImage.getAction() == KittyImage.ACTION__TRANSMIT_AND_DISPLAY) { + placeKittyImage(kittyImage, storedImage); + } + } + + /** + * Do a kitty graphics `a=p` command to display an image that was already transmitted with an + * `a=t` or `a=T` command without transmitting the image data again. + */ + private void doKittyGraphicsPut(KittyImage kittyImage) { + long imageId = kittyImage.getImageId(); + if (imageId == KittyImage.IMAGE_ID__NONE) { + kittyImage.setStateFailed(KittyImage.ERROR__EINVAL, "missing image id"); + return; + } + + KittyStoredImage storedImage = mKittyImages.get(imageId); + if (storedImage == null) { + kittyImage.setStateFailed(KittyImage.ERROR__ENOENT, "no image transmitted for image id"); + return; + } + + placeKittyImage(kittyImage, storedImage); + } + + /** Display a kitty graphics image at the cursor position as a placement of its image id. */ + private void placeKittyImage(KittyImage kittyImage, KittyStoredImage storedImage) { + // The source rectangle is validated against the dimensions the image was transmitted with if + // they are known, so that a client gets an error response instead of the wrong part of the + // image being displayed. + if (!kittyImage.validateSourceRectangle(storedImage.mPixelWidth, storedImage.mPixelHeight)) { + return; + } + + int[] cursorDelta = mScreen.addTerminalBitmapForKittyImage(storedImage.mFormat, storedImage.mImage, + storedImage.mPixelWidth, storedImage.mPixelHeight, + kittyImage.getSourceX(), kittyImage.getSourceY(), + kittyImage.getSourceWidth(), kittyImage.getSourceHeight(), + mCursorCol, mCursorRow, mCellWidthPixels, mCellHeightPixels, + kittyImage.getWidthPixels(mCellWidthPixels), kittyImage.getHeightPixels(mCellHeightPixels), + kittyImage.shouldPreserveAspectRatio(), + kittyImage.getImageId(), kittyImage.getPlacementId()); + + // A bitmap always covers at least one column if it was created successfully. + if (cursorDelta[1] < 1) { + kittyImage.setStateFailed(KittyImage.ERROR__EBADPNG, "displaying image failed"); + return; + } + + // The `C=1` key requires the cursor to be left where it was, which clients that position the + // cursor themselves before displaying an image rely on. + if (kittyImage.shouldMoveCursor()) { + moveCursorAfterTerminalBitmap(cursorDelta); + } + } + + /** Do a kitty graphics `a=d` command. */ + private void doKittyGraphicsDelete(KittyImage kittyImage) { + char deleteMode = kittyImage.getDeleteMode(); + switch (deleteMode) { + case KittyImage.DELETE_MODE__NONE: // The `d` key defaults to `d=a`. + case KittyImage.DELETE_MODE__ALL: + case KittyImage.DELETE_MODE__ALL_AND_FREE_DATA: + // All the placements are deleted regardless of the image id passed as per the protocol. + mScreen.deleteAllKittyImagePlacements(); + if (deleteMode == KittyImage.DELETE_MODE__ALL_AND_FREE_DATA) { + clearKittyImages(); + } + break; + case KittyImage.DELETE_MODE__ID: + case KittyImage.DELETE_MODE__ID_AND_FREE_DATA: + long imageId = kittyImage.getImageId(); + if (imageId == KittyImage.IMAGE_ID__NONE) { + kittyImage.setStateFailed(KittyImage.ERROR__EINVAL, "missing image id"); + return; + } + + // Deleting an image that does not exist is not an error as per the protocol. Only + // the placement passed with the `p` key is deleted if it is passed. + mScreen.deleteKittyImagePlacements(imageId, kittyImage.getPlacementId()); + if (deleteMode == KittyImage.DELETE_MODE__ID_AND_FREE_DATA) { + removeKittyImage(imageId); + } + break; + default: + // The `d=n`, `d=c`, `d=p`, `d=q`, `d=r`, `d=x`, `d=y` and `d=z` delete modes require + // image numbers or coordinates, which are not supported. + kittyImage.setStateFailed(KittyImage.ERROR__ENOTSUP, "unsupported delete mode"); + break; + } + } + + /** + * Send the response for a kitty graphics command as per the quiet level passed with its `q` key. + * + * The success response is `ESC _ G i= ; OK ST` and the error response is + * `ESC _ G i= ; : ST`. + */ + private void respondToKittyGraphicsCommand(KittyImage kittyImage) { + // A response cannot be matched with the command that caused it without an image id, so none + // is sent, like kitty does. + long imageId = kittyImage.getImageId(); + if (imageId == KittyImage.IMAGE_ID__NONE) return; + + int quiet = kittyImage.getQuiet(); + + if (!kittyImage.isFailed()) { + if (quiet >= KittyImage.QUIET__SUCCESS) return; + mSession.write("\033_Gi=" + imageId + ";OK\033\\"); + } else { + if (quiet >= KittyImage.QUIET__ALL) return; + mSession.write("\033_Gi=" + imageId + ";" + kittyImage.getErrorCode() + ":" + kittyImage.getErrorMessage() + "\033\\"); + } + } + + /** + * Store the image transmitted for a kitty graphics image id so that it can be displayed again + * with an `a=p` command and freed with an `a=d,d=I` command. + */ + private void storeKittyImage(long imageId, KittyStoredImage storedImage) { + removeKittyImage(imageId); + + if (storedImage.mImage.length > KITTY_IMAGES__MAX_TOTAL_SIZE) { + Logger.logWarn(mClient, LOG_TAG, "Not storing kitty graphics image " + imageId + " with" + + " size " + storedImage.mImage.length + " greater than max total size " + KITTY_IMAGES__MAX_TOTAL_SIZE); + return; + } + + mKittyImages.put(imageId, storedImage); + mKittyImagesTotalSize += storedImage.mImage.length; + + // Evict the oldest images if the storage limits are exceeded. + Iterator> iterator = mKittyImages.entrySet().iterator(); + while (iterator.hasNext() && + (mKittyImages.size() > KITTY_IMAGES__MAX_COUNT || mKittyImagesTotalSize > KITTY_IMAGES__MAX_TOTAL_SIZE)) { + Map.Entry entry = iterator.next(); + if (entry.getKey() == imageId) continue; + mKittyImagesTotalSize -= entry.getValue().mImage.length; + iterator.remove(); + } + } + + /** Remove the image stored for a kitty graphics image id. */ + private void removeKittyImage(long imageId) { + KittyStoredImage storedImage = mKittyImages.remove(imageId); + if (storedImage != null) { + mKittyImagesTotalSize -= storedImage.mImage.length; + } + } + + /** Remove the image data stored for all the kitty graphics image ids. */ + private void clearKittyImages() { + mKittyImages.clear(); + mKittyImagesTotalSize = 0; + } + + /** + * Get the length of the image data stored for a kitty graphics image id, or `-1` if no image + * data is stored for it. + */ + public int getKittyImageDataLength(long imageId) { + KittyStoredImage storedImage = mKittyImages.get(imageId); + return storedImage != null ? storedImage.mImage.length : -1; + } + + /** + * Move the cursor to after a {@link TerminalBitmap} that was added to the screen. + * + * If the image fits on the screen, then the cursor is moved to the column after the image on the + * last row the image covers, otherwise it is wrapped to the first column of the row after the + * image. + * + * **Note that this changes the behaviour of sixel and iTerm images as well.** The condition to + * check if the image fits was `col < mColumns - 1`, which treated an image that ends exactly at + * the last column of the screen as if it did not fit, and so consumed one row more than the image + * covers and left the cursor at the first column of it. An image that ends exactly at the last + * column does fit, so the condition must be `col < mColumns`. Check the `0003` patch in the pull + * request for this as a standalone change. + * + * @param cursorDelta The cursor delta returned by + * {@link TerminalBuffer#addTerminalBitmapForImage(byte[], int, int, int, int, int, int, boolean)}, + * where the first value is the number of rows the bitmap covers on the screen and the second + * value is the number of columns. The array is not modified, since it is the + * {@link TerminalBitmap#getCursorDelta()} array of the bitmap itself. + */ + void moveCursorAfterTerminalBitmap(int[] cursorDelta) { + int rows = cursorDelta[0]; + int col = cursorDelta[1] + mCursorCol; + if (col < mColumns) { + rows -= 1; + } else { + col = 0; + } + for (; rows > 0; rows--) { + doLinefeed(); + } + mCursorCol = col; + } @@ -2974,16 +3551,7 @@ else if (controlCommandPrefix.equals("FileEnd")) { iTermImage.getWidth(), iTermImage.getHeight(), iTermImage.shouldPreserveAspectRatio()); - int col = cursorDelta[1] + mCursorCol; - if (col < mColumns) { - cursorDelta[0] -= 1; - } else { - col = 0; - } - for (; cursorDelta[0] > 0; cursorDelta[0]--) { - doLinefeed(); - } - mCursorCol = col; + moveCursorAfterTerminalBitmap(cursorDelta); } // Saving files in downloads folder is not supported currently. else {} @@ -3503,7 +4071,10 @@ public void reset() { clearTerminalControlArgs(); clearOscTypeVariables(); + clearApcTypeVariables(); mITermImage = null; + mKittyImage = null; + clearKittyImages(); } public String getSelectedText(int x1, int y1, int x2, int y2) { diff --git a/terminal-emulator/src/test/java/com/termux/terminal/ApcTest.java b/terminal-emulator/src/test/java/com/termux/terminal/ApcTest.java index 4f6292aaa9..954a308a0b 100644 --- a/terminal-emulator/src/test/java/com/termux/terminal/ApcTest.java +++ b/terminal-emulator/src/test/java/com/termux/terminal/ApcTest.java @@ -5,9 +5,9 @@ public class ApcTest extends TerminalTestCase { public void testApcConsumed() { // At time of writing this is part of what yazi sends for probing for kitty graphics protocol support: // https://github.com/sxyazi/yazi/blob/0cdaff98d0b3723caff63eebf1974e7907a43a2c/yazi-adapter/src/emulator.rs#L129 - // This should not result in anything being written to the screen: If kitty graphics protocol support - // is implemented it should instead result in an error code on stdin, and if not it should be consumed - // silently just as xterm does. See https://sw.kovidgoyal.net/kitty/graphics-protocol/. + // This should not result in anything being written to the screen: The kitty graphics protocol + // capability query is instead answered on stdin, which is checked by `KittyGraphicsTest`. + // See https://sw.kovidgoyal.net/kitty/graphics-protocol/. withTerminalSized(2, 2) .enterString("\033_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\033\\") .assertLinesAre(" ", " "); diff --git a/terminal-emulator/src/test/java/com/termux/terminal/KittyGraphicsTest.java b/terminal-emulator/src/test/java/com/termux/terminal/KittyGraphicsTest.java new file mode 100644 index 0000000000..a14db7ace4 --- /dev/null +++ b/terminal-emulator/src/test/java/com/termux/terminal/KittyGraphicsTest.java @@ -0,0 +1,1315 @@ +package com.termux.terminal; + +import java.util.Arrays; +import java.util.Base64; +import java.util.HashSet; +import java.util.Set; + +/** + * Tests for the kitty graphics protocol `APC _G` commands. + * + * - https://sw.kovidgoyal.net/kitty/graphics-protocol/ + * + * Note that the android `Bitmap` and `BitmapFactory` classes are not available in unit tests since + * `unitTests.returnDefaultValues` is enabled for the `terminal-emulator` module, which makes + * `Bitmap.createBitmap()` return `null`, so no image can ever actually be placed on the screen here. + * Commands that display an image therefore always respond with an `EBADPNG` error in these tests, + * while the same commands respond with `OK` on a device. Everything up to creating the bitmap does + * not depend on the android platform and is covered, and the code that operates on placements after + * they have been created is covered by creating the placements directly instead, check + * {@link #testDeleteKittyImagePlacementsClearsTheCellsOfThePlacements()}. + */ +public class KittyGraphicsTest extends TerminalTestCase { + + /** A valid `1x1` `PNG` image, `base64` encoded, which decodes to 70 bytes. */ + private static final String PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg=="; + + /** The length of the image data {@link #PNG_BASE64} decodes to. */ + private static final int PNG_LENGTH = 70; + + /** The first chunk of {@link #PNG_BASE64}, whose length is a multiple of `4` as required. */ + private static final String PNG_BASE64_CHUNK_0 = PNG_BASE64.substring(0, 40); + private static final String PNG_BASE64_CHUNK_1 = PNG_BASE64.substring(40); + + /** Raw `RGB` (`f=24`) pixel data for a `1x1` and a `2x2` image, `base64` encoded. */ + private static final String RGB_BASE64_1X1 = "AQID"; + private static final String RGB_BASE64_2X2 = "AQIDBAUGBwgJCgsM"; + + /** Raw `RGBA` (`f=32`) pixel data for a `1x1` and a `2x2` image, `base64` encoded. */ + private static final String RGBA_BASE64_1X1 = "AQIDBA=="; + private static final String RGBA_BASE64_2X2 = "AQIDBAUGBwgJCgsMDQ4PEA=="; + + + + /* Capability queries (`a=q`). */ + + public void testCapabilityQueryRespondsOk() { + // The exact string yazi and the kitty documentation use for probing terminal support. + withTerminalSized(10, 5); + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\033\\", + "\033_Gi=31;OK\033\\"); + } + + public void testCapabilityQueryDoesNotPaintScreen() { + withTerminalSized(2, 2); + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\033\\", + "\033_Gi=31;OK\033\\"); + assertLinesAre(" ", " "); + } + + public void testCapabilityQueryWithoutImageIdIsSilent() { + // A response cannot be matched with the command that caused it without an image id. + withTerminalSized(10, 5); + assertEnteringStringGivesResponse("\033_Ga=q,t=d,f=24;AAAA\033\\", ""); + } + + public void testCapabilityQueryWithUnsupportedTransmissionMediumRespondsError() { + withTerminalSized(10, 5); + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=f,f=24;L3RtcC9pbWc=\033\\", + "\033_Gi=31;ENOTSUP:unsupported transmission medium\033\\"); + } + + public void testCapabilityQueryValidatesTheImageData() { + // A capability query must not blindly respond with `OK`, since a client uses the response to + // decide what it can send. + withTerminalSized(10, 5); + + // The pixel dimensions of the raw formats must match the length of the image data. + assertEnteringStringGivesResponse( + "\033_Gi=31,s=2,v=2,a=q,t=d,f=24;AAAA\033\\", + "\033_Gi=31;EINVAL:image data of length 3 does not match the 12 bytes required for pixel dimensions 2x2\033\\"); + + assertEnteringStringGivesResponse( + "\033_Gi=31,a=q,t=d,f=32;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=31;EINVAL:missing pixel dimensions for image data of length 4\033\\"); + + // Compressed image data is not supported. + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=d,f=24,o=z;AAAA\033\\", + "\033_Gi=31;ENOTSUP:unsupported compression\033\\"); + + // The image data must be valid `base64`. + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=d,f=24;AA!A\033\\", + "\033_Gi=31;EBADDATA:invalid base64 image data\033\\"); + } + + public void testCapabilityQueryDoesNotStoreTheImage() { + withTerminalSized(10, 5); + assertEnteringStringGivesResponse( + "\033_Gi=31,s=1,v=1,a=q,t=d,f=24;" + RGB_BASE64_1X1 + "\033\\", + "\033_Gi=31;OK\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(31)); + } + + + /* Quiet levels (`q=`). */ + + public void testQuietLevelsForSuccessResponse() { + withTerminalSized(10, 5); + + // `q` not passed, so respond. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=1,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=1;OK\033\\"); + + // `q=1` suppresses success responses. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=1,m=0,q=1;" + PNG_BASE64 + "\033\\", ""); + + // `q=2` suppresses success and error responses. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=1,m=0,q=2;" + PNG_BASE64 + "\033\\", ""); + } + + public void testQuietLevelsForErrorResponse() { + withTerminalSized(10, 5); + + // `q` not passed, so respond. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=2,m=0,o=z;" + PNG_BASE64 + "\033\\", + "\033_Gi=2;ENOTSUP:unsupported compression\033\\"); + + // `q=1` only suppresses success responses, so still respond. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=2,m=0,o=z,q=1;" + PNG_BASE64 + "\033\\", + "\033_Gi=2;ENOTSUP:unsupported compression\033\\"); + + // `q=2` suppresses success and error responses. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=2,m=0,o=z,q=2;" + PNG_BASE64 + "\033\\", ""); + } + + public void testResponseDoesNotDependOnTheOrderOfTheControlDataKeys() { + // All the control data is read even if a key is invalid, so that the `i` and `q` keys + // required to send the response are read regardless of the order the keys are passed in. + withTerminalSized(10, 5); + + assertEnteringStringGivesResponse("\033_Gq=2,o=z,i=3,a=t,f=100,m=0;" + PNG_BASE64 + "\033\\", ""); + assertEnteringStringGivesResponse("\033_Go=z,q=2,i=3,a=t,f=100,m=0;" + PNG_BASE64 + "\033\\", ""); + assertEnteringStringGivesResponse("\033_Go=z,i=3,q=2,a=t,f=100,m=0;" + PNG_BASE64 + "\033\\", ""); + + assertEnteringStringGivesResponse("\033_Go=z,i=3,a=t,f=100,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=3;ENOTSUP:unsupported compression\033\\"); + } + + + /* Transmitting images (`a=t`). */ + + public void testTransmitStoresImageData() { + withTerminalSized(10, 5); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=424242,m=0,q=2;" + PNG_BASE64 + "\033\\", ""); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(424242)); + } + + public void testTransmitWithoutImageIdDoesNotStoreImageData() { + withTerminalSized(10, 5); + assertEnteringStringGivesResponse("\033_Ga=t,f=100,t=d,m=0;" + PNG_BASE64 + "\033\\", ""); + assertEquals(-1, mTerminal.getKittyImageDataLength(0)); + } + + public void testTransmitDoesNotPaintScreen() { + withTerminalSized(4, 2) + .enterString("\033_Ga=t,f=100,t=d,i=1,m=0,q=2;" + PNG_BASE64 + "\033\\") + .enterString("ok") + .assertLinesAre("ok ", " "); + } + + public void testTransmitAndDisplayDoesNotCrashWhenBitmapCannotBeCreated() { + // The android `BitmapFactory` is stubbed out in unit tests, so the image cannot be decoded. + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=424242,m=0,c=4,r=2;" + PNG_BASE64 + "\033\\", + "\033_Gi=424242;EBADPNG:displaying image failed\033\\"); + + // The image data is still stored and the screen and cursor are left untouched. + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(424242)); + assertCursorAt(0, 0); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + /* Raw pixel data formats (`f=24` and `f=32`). */ + + public void testTransmitRawRgbAndRgbaImages() { + withTerminalSized(20, 4); + + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=100,s=1,v=1,m=0;" + RGB_BASE64_1X1 + "\033\\", + "\033_Gi=100;OK\033\\"); + assertEquals(3, mTerminal.getKittyImageDataLength(100)); + + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=101,s=2,v=2,m=0;" + RGB_BASE64_2X2 + "\033\\", + "\033_Gi=101;OK\033\\"); + assertEquals(12, mTerminal.getKittyImageDataLength(101)); + + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=102,s=1,v=1,m=0;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=102;OK\033\\"); + assertEquals(4, mTerminal.getKittyImageDataLength(102)); + + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=103,s=2,v=2,m=0;" + RGBA_BASE64_2X2 + "\033\\", + "\033_Gi=103;OK\033\\"); + assertEquals(16, mTerminal.getKittyImageDataLength(103)); + } + + public void testRawImageDataLengthMustMatchThePixelDimensions() { + withTerminalSized(20, 4); + + // `RGB` requires 3 bytes per pixel and `RGBA` requires 4, so data that is too short is + // rejected once all of it has been received. Data that is too long is rejected as soon as it + // arrives instead, check + // `testRawImageWithMoreDataThanTheDeclaredDimensionsIsRejected()`. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=104,s=2,v=2,m=0;" + RGB_BASE64_2X2 + "\033\\", + "\033_Gi=104;EINVAL:image data of length 12 does not match the 16 bytes required for pixel dimensions 2x2\033\\"); + + // The `s` and `v` keys are required for the raw formats. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=104,m=0;" + RGB_BASE64_2X2 + "\033\\", + "\033_Gi=104;EINVAL:missing pixel dimensions for image data of length 12\033\\"); + + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=104,s=2,m=0;" + RGB_BASE64_2X2 + "\033\\", + "\033_Gi=104;EINVAL:missing pixel dimensions for image data of length 12\033\\"); + + assertEquals(-1, mTerminal.getKittyImageDataLength(104)); + } + + public void testRawImageWithoutFormatKeyUsesTheProtocolDefaultFormat() { + // The protocol defines `f=32` (raw `RGBA`) as the default if the `f` key is not passed. + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=t,t=d,i=105,s=1,v=1,m=0;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=105;OK\033\\"); + assertEquals(4, mTerminal.getKittyImageDataLength(105)); + } + + public void testRawImageExceedingTheMaxImageDataLengthIsRejected() { + withTerminalSized(20, 4); + + // The pixel dimensions are known before the image data is received for the raw formats, so + // the limit is checked without having to receive all of the data first. The max pixel + // dimensions of `32766x32766` `RGBA` are `4.29GB` of pixel data, which also checks that + // calculating the length does not overflow. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=106,s=32766,v=32766,m=0;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=106;ENOSPC:image data exceeds max length " + KittyImage.IMAGE_DATA__MAX_LENGTH + "\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(106)); + } + + public void testMaxImageDataLengthIsDerivedFromTheMaxBitmapSize() { + // The limit must match the largest bitmap the terminal can draw, so that an image that could + // have been displayed is not rejected while it is being received. The limit is in bytes of + // decoded image data, since the `base64` payload is decoded chunk by chunk instead of being + // collected as text and decoded at the end. + assertEquals(TerminalBitmap.MAX_BITMAP_SIZE, KittyImage.IMAGE_DATA__MAX_LENGTH); + assertTrue(KittyImage.IMAGE_DATA__MAX_LENGTH > 0); + + // A full screen image on a high resolution device must be within the limit, which is + // `1080x2400` raw `RGBA` for a portrait device. + assertTrue(1080 * 2400 * 4 <= KittyImage.IMAGE_DATA__MAX_LENGTH); + } + + /** + * Decode a `base64` payload that is split into chunks with {@link KittyImage} directly, so that + * the decoded bytes themselves can be checked instead of only their length. The chunks are + * processed the same way {@link TerminalEmulator} processes the commands for them. + * + * @return Returns the decoded image data, or `null` if it was rejected. + */ + private byte[] decodeChunkedPayload(String controlData, String... chunks) { + KittyImage kittyImage = null; + + for (int i = 0; i < chunks.length; i++) { + boolean isFinalChunk = i == chunks.length - 1; + StringBuilder apcArgs = new StringBuilder("G" + + (i == 0 ? controlData + "," : "") + "m=" + (isFinalChunk ? 0 : 1) + ";" + chunks[i]); + + KittyImage chunk = new KittyImage(null); + int payloadIndex = chunk.readControlData(apcArgs, /* `G` */ 1); + assertTrue("The control data of chunk " + i + " was rejected", payloadIndex >= 0); + + if (kittyImage == null) { + kittyImage = chunk; + } else { + kittyImage.readContinuationControlData(chunk); + } + + if (!kittyImage.startImage()) return null; + for (int index = payloadIndex; index < apcArgs.length(); index++) { + if (!kittyImage.readImageChar(apcArgs.charAt(index))) return null; + } + } + + return kittyImage.finishImage() ? kittyImage.getDecodedImage() : null; + } + + public void testChunkBoundariesThatAreNotAMultipleOfFourDecodeToTheSameBytes() { + // Each 4 `base64` characters decode to 3 bytes, so a chunk whose length is not a multiple of + // 4 leaves bits that must be carried over to the next chunk. The result is compared against + // the JDK `base64` decoder as an independent reference implementation. + byte[] expected = Base64.getDecoder().decode(PNG_BASE64); + assertEquals(PNG_LENGTH, expected.length); + + // The padding is only valid at the very end of the data, so the offsets that would put it in + // a chunk that is not the final chunk are excluded, which is checked by + // `testBase64PaddingInANonFinalChunkIsRejected()` instead. + int paddingIndex = PNG_BASE64.indexOf('='); + assertTrue(paddingIndex > 0); + + for (int splitIndex = 1; splitIndex < paddingIndex; splitIndex++) { + byte[] decoded = decodeChunkedPayload("a=t,f=100,t=d", + PNG_BASE64.substring(0, splitIndex), PNG_BASE64.substring(splitIndex)); + assertNotNull("Split at index " + splitIndex, decoded); + assertTrue("Split at index " + splitIndex, Arrays.equals(expected, decoded)); + } + + // Three chunks, so that a chunk that is neither the first nor the last carries bits over on + // both of its boundaries. + for (int splitIndex = 2; splitIndex < paddingIndex; splitIndex++) { + byte[] decoded = decodeChunkedPayload("a=t,f=100,t=d", + PNG_BASE64.substring(0, 1), PNG_BASE64.substring(1, splitIndex), PNG_BASE64.substring(splitIndex)); + assertNotNull("Split at indexes 1 and " + splitIndex, decoded); + assertTrue("Split at indexes 1 and " + splitIndex, Arrays.equals(expected, decoded)); + } + + // A chunk of a single character at every offset. + for (int splitIndex = 1; splitIndex < paddingIndex; splitIndex++) { + byte[] decoded = decodeChunkedPayload("a=t,f=100,t=d", + PNG_BASE64.substring(0, splitIndex), PNG_BASE64.substring(splitIndex, splitIndex + 1), + PNG_BASE64.substring(splitIndex + 1)); + assertNotNull("Single character chunk at index " + splitIndex, decoded); + assertTrue("Single character chunk at index " + splitIndex, Arrays.equals(expected, decoded)); + } + } + + public void testChunkBoundariesOfRawPixelDataDecodeToTheSameBytes() { + byte[] expected = Base64.getDecoder().decode(RGB_BASE64_2X2); + assertEquals(12, expected.length); + + // `RGB_BASE64_2X2` has no padding, so every offset can be split at. + for (int splitIndex = 1; splitIndex < RGB_BASE64_2X2.length(); splitIndex++) { + byte[] decoded = decodeChunkedPayload("a=t,f=24,t=d,s=2,v=2", + RGB_BASE64_2X2.substring(0, splitIndex), RGB_BASE64_2X2.substring(splitIndex)); + assertNotNull("Split at index " + splitIndex, decoded); + assertTrue("Split at index " + splitIndex, Arrays.equals(expected, decoded)); + } + } + + public void testChunkBoundariesThroughTheTerminal() { + // The same as above, but through the actual commands, which also checks that the image is + // stored with the assembled length. + int paddingIndex = PNG_BASE64.indexOf('='); + for (int splitIndex = 1; splitIndex < paddingIndex; splitIndex++) { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=400,m=1,q=2;" + PNG_BASE64.substring(0, splitIndex) + "\033\\"); + assertEnteringStringGivesResponse( + "\033_Gm=0,q=2;" + PNG_BASE64.substring(splitIndex) + "\033\\", ""); + assertEquals("Split at index " + splitIndex, PNG_LENGTH, mTerminal.getKittyImageDataLength(400)); + } + + for (int splitIndex = 1; splitIndex < RGBA_BASE64_2X2.indexOf('='); splitIndex++) { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=32,t=d,i=401,s=2,v=2,m=1,q=2;" + RGBA_BASE64_2X2.substring(0, splitIndex) + "\033\\"); + enterString("\033_Gm=0,q=2;" + RGBA_BASE64_2X2.substring(splitIndex) + "\033\\"); + assertEquals("Split at index " + splitIndex, 16, mTerminal.getKittyImageDataLength(401)); + } + } + + public void testBase64PaddingInANonFinalChunkIsRejected() { + withTerminalSized(20, 4); + + // The padding is only valid at the very end of the `base64` data, so a chunk that is not the + // final chunk must not contain any. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=402,s=1,v=1,m=1;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=402;EBADDATA:base64 padding in a chunk that is not the final chunk\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(402)); + + // The same data is accepted if it is the final chunk. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=402,s=1,v=1,m=0;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=402;OK\033\\"); + assertEquals(4, mTerminal.getKittyImageDataLength(402)); + } + + public void testBase64DataAfterThePaddingIsRejected() { + withTerminalSized(20, 4); + + // Within a single chunk. + assertEnteringStringGivesResponse("\033_Ga=t,f=24,t=d,i=403,s=1,v=1,m=0;QQ==QQ==\033\\", + "\033_Gi=403;EBADDATA:base64 data after the padding\033\\"); + + // And across chunks, where the first chunk ended with the padding. + enterString("\033_Ga=t,f=24,t=d,i=403,s=1,v=1,m=1;AQI\033\\"); + assertEnteringStringGivesResponse("\033_Gm=0;D" + "\033\\", "\033_Gi=403;OK\033\\"); + } + + public void testInvalidBase64InAnyChunkIsRejected() { + withTerminalSized(20, 4); + + // The strict validation must be kept for every chunk, not only the final one. + assertEnteringStringGivesResponse("\033_Ga=t,f=100,t=d,i=404,m=1;AA!A\033\\", + "\033_Gi=404;EBADDATA:invalid base64 image data\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(404)); + + // A total number of characters that leaves a single character in the final quad. + enterString("\033_Ga=t,f=100,t=d,i=404,m=1;AAAA\033\\"); + assertEnteringStringGivesResponse("\033_Gm=0;A\033\\", + "\033_Gi=404;EBADDATA:invalid base64 image data\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(404)); + } + + public void testRawImageWithMoreDataThanTheDeclaredDimensionsIsRejected() { + withTerminalSized(20, 4); + + // The extra data is rejected as soon as it arrives, so that the memory for it is never + // allocated for an image that cannot be displayed anyway. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=405,s=2,v=2,m=0;" + RGBA_BASE64_2X2 + "\033\\", + "\033_Gi=405;EINVAL:image data exceeds the 12 bytes required for pixel dimensions 2x2\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(405)); + } + + public void testRawImageMultiChunkTransmitAssemblesPayload() { + withTerminalSized(20, 4); + + // `RGB_BASE64_2X2` split at 8 characters, which is a multiple of 4 as required. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=24,t=d,i=107,s=2,v=2,m=1;" + RGB_BASE64_2X2.substring(0, 8) + "\033\\", ""); + assertEnteringStringGivesResponse( + "\033_Gm=0;" + RGB_BASE64_2X2.substring(8) + "\033\\", "\033_Gi=107;OK\033\\"); + assertEquals(12, mTerminal.getKittyImageDataLength(107)); + } + + + public void testTransmitAndDisplayWithOnlyColumnsDoesNotHang() { + // A `c` key without an `r` key results in the aspect ratio of the image being preserved, + // which requires the image dimensions to be known. + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=1,m=0,c=4;" + PNG_BASE64 + "\033\\", + "\033_Gi=1;EBADPNG:displaying image failed\033\\"); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=1,m=0,r=2;" + PNG_BASE64 + "\033\\", + "\033_Gi=1;EBADPNG:displaying image failed\033\\"); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=1,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=1;EBADPNG:displaying image failed\033\\"); + } + + + /* Chunked transmission (`m=1`/`m=0`). */ + + public void testMultiChunkTransmitAssemblesPayload() { + withTerminalSized(20, 4); + + // No response and nothing stored until the final chunk is received. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=7,m=1,q=2;" + PNG_BASE64_CHUNK_0 + "\033\\", ""); + assertEquals(-1, mTerminal.getKittyImageDataLength(7)); + + assertEnteringStringGivesResponse("\033_Gm=0,q=2;" + PNG_BASE64_CHUNK_1 + "\033\\", ""); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(7)); + } + + public void testMultiChunkTransmitRespondsOnlyAfterFinalChunk() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=7,m=1;" + PNG_BASE64_CHUNK_0 + "\033\\", ""); + assertEnteringStringGivesResponse( + "\033_Gm=0;" + PNG_BASE64_CHUNK_1 + "\033\\", "\033_Gi=7;OK\033\\"); + } + + public void testMultiChunkTransmitWithoutFinalChunkIsNotProcessed() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=7,m=1;" + PNG_BASE64_CHUNK_0 + "\033\\", ""); + + // The image is never completed, so it is never stored and no response is ever sent. + assertEquals(-1, mTerminal.getKittyImageDataLength(7)); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + assertEquals("", mOutput.getOutputAndClear()); + } + + public void testInterleavedTextDuringChunkedTransferDoesNotCorruptState() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=7,m=1,q=2;" + PNG_BASE64_CHUNK_0 + "\033\\"); + + // Printing text between the chunks of an image is a protocol violation, but the text must + // still be printed and the image must still be received correctly. + enterString("hello"); + enterString("\033_Gm=0,q=2;" + PNG_BASE64_CHUNK_1 + "\033\\"); + + assertLinesAre("hello ", " ", + " ", " "); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(7)); + assertEquals("", mOutput.getOutputAndClear()); + } + + public void testInterleavedEscapeSequenceDuringChunkedTransferDoesNotCorruptState() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=7,m=1,q=2;" + PNG_BASE64_CHUNK_0 + "\033\\"); + + // A `CSI` sequence, an unsupported `APC` command and an `OSC` command between the chunks. + enterString("\033[2;3H\033_unrelated\033\\\033]0;title\033\\"); + enterString("\033_Gm=0,q=2;" + PNG_BASE64_CHUNK_1 + "\033\\"); + + assertCursorAt(1, 2); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(7)); + assertEquals("", mOutput.getOutputAndClear()); + } + + public void testNewCommandDuringChunkedTransferDropsIncompleteImage() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=7,m=1,q=2;" + PNG_BASE64_CHUNK_0 + "\033\\"); + + // A command that passes the `a` key is always a new command, never a continuation chunk. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=8,m=0,q=2;" + PNG_BASE64 + "\033\\", ""); + + assertEquals(-1, mTerminal.getKittyImageDataLength(7)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(8)); + } + + public void testContinuationChunkWithUnexpectedKeysIgnoresThemGracefully() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=7,m=1,q=2;" + PNG_BASE64_CHUNK_0 + "\033\\"); + + // A continuation chunk that re-passes keys is a protocol violation. Since it passes the `a` + // key it is treated as a new command, which drops the incomplete image, and the payload of + // the new command is not valid image data on its own. The `f` key is not passed either, so + // the protocol default of raw `RGBA` pixel data applies, whose dimensions are missing. + assertEnteringStringGivesResponse( + "\033_Ga=t,i=999,m=0;" + PNG_BASE64_CHUNK_1 + "\033\\", + "\033_Gi=999;EINVAL:missing pixel dimensions for image data of length 40\033\\"); + + assertEquals(-1, mTerminal.getKittyImageDataLength(7)); + assertEquals(-1, mTerminal.getKittyImageDataLength(999)); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testContinuationChunkWithoutPendingImageIsIgnored() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse("\033_Gm=0,q=2;" + PNG_BASE64 + "\033\\", ""); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + + /* Invalid payloads and control data. */ + + public void testInvalidBase64PayloadRespondsError() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=9,m=0;AA!AAAAA\033\\", + "\033_Gi=9;EBADDATA:invalid base64 image data\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(9)); + + // A `base64` string of a length that cannot encode any byte is invalid as well. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=9,m=0;AAAAA\033\\", + "\033_Gi=9;EBADDATA:invalid base64 image data\033\\"); + + // An empty payload cannot be decoded into an image. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=9,m=0;\033\\", + "\033_Gi=9;EBADDATA:empty image data\033\\"); + } + + public void testInvalidBase64PaddingIsRejected() { + withTerminalSized(20, 4); + + // More than two padding characters, and a total length that is not a multiple of four. + assertEnteringStringGivesResponse("\033_Ga=t,f=24,t=d,i=9,s=1,v=1,m=0;QQ====\033\\", + "\033_Gi=9;EBADDATA:invalid base64 image data\033\\"); + assertEnteringStringGivesResponse("\033_Ga=t,f=24,t=d,i=9,s=1,v=1,m=0;QQ=\033\\", + "\033_Gi=9;EBADDATA:invalid base64 image data\033\\"); + assertEnteringStringGivesResponse("\033_Ga=t,f=24,t=d,i=9,s=1,v=1,m=0;=\033\\", + "\033_Gi=9;EBADDATA:invalid base64 image data\033\\"); + + // Valid padding, and the unpadded form which clients are allowed to send. + assertEnteringStringGivesResponse("\033_Ga=t,f=32,t=d,i=9,s=1,v=1,m=0;" + RGBA_BASE64_1X1 + "\033\\", + "\033_Gi=9;OK\033\\"); + assertEnteringStringGivesResponse("\033_Ga=t,f=32,t=d,i=9,s=1,v=1,m=0;AQIDBA\033\\", + "\033_Gi=9;OK\033\\"); + } + + public void testMalformedControlDataIsRejected() { + withTerminalSized(20, 4); + + // A key without a `=` and a value. The image id was read before the malformed key, so it + // can be used to match the error response with the command. + assertEnteringStringGivesResponse("\033_Ga=t,i=10,X;" + PNG_BASE64 + "\033\\", + "\033_Gi=10;EINVAL:malformed control data\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(10)); + + // The remaining keys are read even if the malformed key comes before them. + assertEnteringStringGivesResponse("\033_GX,i=10;" + PNG_BASE64 + "\033\\", + "\033_Gi=10;EINVAL:malformed control data\033\\"); + assertEnteringStringGivesResponse("\033_GX,i=10,q=2;" + PNG_BASE64 + "\033\\", ""); + + // A key with an empty value. + assertEnteringStringGivesResponse("\033_Ga=t,i=10,c=;" + PNG_BASE64 + "\033\\", + "\033_Gi=10;EINVAL:invalid columns\033\\"); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testUnknownActionRespondsError() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse("\033_Ga=X,i=11;\033\\", + "\033_Gi=11;ENOTSUP:unsupported action\033\\"); + + // The `a=f` (frame), `a=a` (animate) and `a=c` (compose) actions are valid protocol actions, + // but are not supported. + assertEnteringStringGivesResponse("\033_Ga=f,i=11;\033\\", + "\033_Gi=11;ENOTSUP:unsupported action\033\\"); + assertEnteringStringGivesResponse("\033_Ga=a,i=11;\033\\", + "\033_Gi=11;ENOTSUP:unsupported action\033\\"); + assertEnteringStringGivesResponse("\033_Ga=c,i=11;\033\\", + "\033_Gi=11;ENOTSUP:unsupported action\033\\"); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + + /* Displaying an already transmitted image (`a=p`). */ + + public void testPutUsesTheStoredImageWithoutRetransmittingIt() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=200,m=0,q=2;" + PNG_BASE64 + "\033\\"); + + // The image data is not sent again, so a failure to display it can only come from the stored + // image being looked up and used. Creating the bitmap always fails in unit tests. + assertEnteringStringGivesResponse("\033_Ga=p,i=200,p=1,c=4,r=2,z=0,C=1\033\\", + "\033_Gi=200;EBADPNG:displaying image failed\033\\"); + + // The stored image is kept so that it can be displayed again. + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(200)); + } + + public void testPutForUnknownImageIdRespondsError() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse("\033_Ga=p,i=201,p=1,c=4,r=2,C=1\033\\", + "\033_Gi=201;ENOENT:no image transmitted for image id\033\\"); + + // The image data is freed by a `d=I` delete, after which it cannot be displayed again. + enterString("\033_Ga=t,f=100,t=d,i=201,m=0,q=2;" + PNG_BASE64 + "\033\\"); + enterString("\033_Ga=d,d=I,i=201,q=2\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=201,p=1,C=1\033\\", + "\033_Gi=201;ENOENT:no image transmitted for image id\033\\"); + } + + public void testPutWithoutImageIdIsIgnored() { + withTerminalSized(20, 4); + // No response can be sent without an image id, so only check that nothing breaks. + assertEnteringStringGivesResponse("\033_Ga=p,p=1,c=4,r=2,C=1\033\\", ""); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + /* Source rectangles (`x=`, `y=`, `w=` and `h=`). */ + + public void testSourceRectangleWithinTheImageIsAccepted() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=32,t=d,i=300,s=2,v=2,m=0,q=2;" + RGBA_BASE64_2X2 + "\033\\"); + + // Creating the bitmap always fails in unit tests, so reaching `EBADPNG` means the source + // rectangle passed validation and the placement was attempted. + assertEnteringStringGivesResponse("\033_Ga=p,i=300,p=1,x=0,y=0,w=1,h=1,c=1,r=1,C=1\033\\", + "\033_Gi=300;EBADPNG:displaying image failed\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=300,p=1,x=1,y=1,w=1,h=1,c=1,r=1,C=1\033\\", + "\033_Gi=300;EBADPNG:displaying image failed\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=300,p=1,x=0,y=0,w=2,h=2,c=2,r=2,C=1\033\\", + "\033_Gi=300;EBADPNG:displaying image failed\033\\"); + + // The `w` and `h` keys default to the rest of the image after the `x` and `y` offset. + assertEnteringStringGivesResponse("\033_Ga=p,i=300,p=1,x=1,y=1,c=1,r=1,C=1\033\\", + "\033_Gi=300;EBADPNG:displaying image failed\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=300,p=1,x=1,c=1,r=2,C=1\033\\", + "\033_Gi=300;EBADPNG:displaying image failed\033\\"); + } + + public void testSourceRectangleOutsideTheImageIsRejected() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=32,t=d,i=301,s=2,v=2,m=0,q=2;" + RGBA_BASE64_2X2 + "\033\\"); + + // The width extends past the right edge of the image. + assertEnteringStringGivesResponse("\033_Ga=p,i=301,p=1,x=1,y=0,w=2,h=1,c=1,r=1,C=1\033\\", + "\033_Gi=301;EINVAL:source rectangle x 1 and width 2 exceed image width 2\033\\"); + + // The height extends past the bottom edge of the image. + assertEnteringStringGivesResponse("\033_Ga=p,i=301,p=1,x=0,y=1,w=1,h=2,c=1,r=1,C=1\033\\", + "\033_Gi=301;EINVAL:source rectangle y 1 and height 2 exceed image height 2\033\\"); + + // The offset itself is outside the image. + assertEnteringStringGivesResponse("\033_Ga=p,i=301,p=1,x=2,y=0,c=1,r=1,C=1\033\\", + "\033_Gi=301;EINVAL:source rectangle x 2 is outside image width 2\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=301,p=1,x=0,y=2,c=1,r=1,C=1\033\\", + "\033_Gi=301;EINVAL:source rectangle y 2 is outside image height 2\033\\"); + + // A source rectangle far larger than the image. + assertEnteringStringGivesResponse("\033_Ga=p,i=301,p=1,x=0,y=0,w=100,h=100,c=4,r=4,C=1\033\\", + "\033_Gi=301;EINVAL:source rectangle x 0 and width 100 exceed image width 2\033\\"); + + // The image is not deleted by a rejected placement. + assertEquals(16, mTerminal.getKittyImageDataLength(301)); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testSourceRectangleIsValidatedForTransmitAndDisplayAsWell() { + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=24,t=d,i=302,s=2,v=2,x=1,y=0,w=2,h=1,c=1,r=1,m=0;" + RGB_BASE64_2X2 + "\033\\", + "\033_Gi=302;EINVAL:source rectangle x 1 and width 2 exceed image width 2\033\\"); + + // The image is still transmitted and stored even though the placement was rejected. + assertEquals(12, mTerminal.getKittyImageDataLength(302)); + } + + public void testSourceRectangleKeyValuesAreValidated() { + withTerminalSized(20, 4); + + // The `w` and `h` keys cannot be `0`, since a source rectangle must cover at least one pixel. + assertEnteringStringGivesResponse("\033_Ga=p,i=303,w=0\033\\", + "\033_Gi=303;EINVAL:invalid source rectangle width\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=303,h=0\033\\", + "\033_Gi=303;EINVAL:invalid source rectangle height\033\\"); + + // Negative and out of range values. + assertEnteringStringGivesResponse("\033_Ga=p,i=303,x=-1\033\\", + "\033_Gi=303;EINVAL:invalid source rectangle x\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=303,y=99999\033\\", + "\033_Gi=303;EINVAL:invalid source rectangle y\033\\"); + } + + public void testSourceRectangleIsNotValidatedIfTheImageDimensionsAreNotKnown() { + // A `PNG` image can be transmitted without the `s` and `v` keys, in which case its dimensions + // are only known once it has been decoded, so the source rectangle can only be validated + // against the bitmap while creating it, which always fails in unit tests. + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=304,m=0,q=2;" + PNG_BASE64 + "\033\\"); + assertEnteringStringGivesResponse("\033_Ga=p,i=304,p=1,x=0,y=0,w=100,h=100,c=4,r=4,C=1\033\\", + "\033_Gi=304;EBADPNG:displaying image failed\033\\"); + } + + + public void testPutDoesNotMoveTheCursorForCursorMovementPolicyOne() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=202,m=0,q=2;" + PNG_BASE64 + "\033\\"); + enterString("\033[2;3H"); + assertCursorAt(1, 2); + + // This is the sequence herdr sends, which positions the cursor itself beforehand. + enterString("\033_Ga=p,i=202,p=1,c=4,r=2,z=0,C=1,q=2\033\\"); + assertCursorAt(1, 2); + assertEquals("", mOutput.getOutputAndClear()); + } + + public void testUnknownKeysAreIgnored() { + withTerminalSized(20, 4); + + // The `z` (z-index), `U` (unicode placeholder), `X` and `Y` (cell pixel offset) keys are + // ignored, as are keys that are not part of the protocol at all. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=12,m=0,z=1,U=1,X=3,Y=4,Z=9;" + PNG_BASE64 + "\033\\", + "\033_Gi=12;OK\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(12)); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testUnsupportedTransmissionMediumsAndCompressionRespondError() { + withTerminalSized(20, 4); + + // Reading image data from a file, a temporary file or shared memory is not supported. + assertEnteringStringGivesResponse("\033_Ga=T,f=100,t=f,i=13,m=0;L3RtcC9pbWc=\033\\", + "\033_Gi=13;ENOTSUP:unsupported transmission medium\033\\"); + assertEnteringStringGivesResponse("\033_Ga=T,f=100,t=t,i=13,m=0;L3RtcC9pbWc=\033\\", + "\033_Gi=13;ENOTSUP:unsupported transmission medium\033\\"); + assertEnteringStringGivesResponse("\033_Ga=T,f=100,t=s,i=13,m=0;L3RtcC9pbWc=\033\\", + "\033_Gi=13;ENOTSUP:unsupported transmission medium\033\\"); + + // `zlib` compressed image data is not supported. + assertEnteringStringGivesResponse("\033_Ga=T,f=100,t=d,o=z,i=13,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=13;ENOTSUP:unsupported compression\033\\"); + + // An image format that is not part of the protocol at all. + assertEnteringStringGivesResponse("\033_Ga=T,f=50,t=d,i=13,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=13;EINVAL:invalid image format\033\\"); + + assertEquals(-1, mTerminal.getKittyImageDataLength(13)); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testImageIdRangeIsValidated() { + withTerminalSized(20, 4); + + // Image ids are 32-bit unsigned integers, so the max value is accepted. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=4294967295,m=0;" + PNG_BASE64 + "\033\\", + "\033_Gi=4294967295;OK\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(4294967295L)); + + // One more than the max value is rejected. No response is sent since the image id required + // to match the response with the command could not be read. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=4294967296,m=0;" + PNG_BASE64 + "\033\\", ""); + assertEquals(-1, mTerminal.getKittyImageDataLength(4294967296L)); + + // A value with more digits than any 32-bit unsigned integer must not overflow. + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=99999999999999999999,m=0;" + PNG_BASE64 + "\033\\", ""); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=100,t=d,i=-1,m=0;" + PNG_BASE64 + "\033\\", ""); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testExtremePlacementSizeIsRejected() { + withTerminalSized(20, 4); + + // The x and y coordinates of a bitmap cell are stored in 12 bits each, so a placement + // cannot be larger than `4095` cells in either direction. + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=14,m=0,c=4096,r=1;" + PNG_BASE64 + "\033\\", + "\033_Gi=14;EINVAL:invalid columns\033\\"); + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=14,m=0,c=1,r=99999;" + PNG_BASE64 + "\033\\", + "\033_Gi=14;EINVAL:invalid rows\033\\"); + + // A placement that is within the limits but far larger than the screen and the max bitmap + // size must be handled without crashing. + assertEnteringStringGivesResponse( + "\033_Ga=T,f=100,t=d,i=14,m=0,c=4095,r=4095;" + PNG_BASE64 + "\033\\", + "\033_Gi=14;EBADPNG:displaying image failed\033\\"); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testSingleCommandWithLargeImageDataIsSupported() { + // The protocol recommends that clients split the image data into chunks of at most 4096 + // bytes, but that is not a requirement, and a client may send an entire image with a single + // command. The image data is decoded as it is received, so the size is not limited. + // This is `128x128` raw `RGBA`, which is `65536` bytes of image data and `87384` characters + // of `base64`, far more than the args buffer of a command can hold. + StringBuilder payload = new StringBuilder(); + for (int i = 0; i < 21845; i++) { + payload.append("AAAA"); + } + payload.append("AA=="); + assertEquals(87384, payload.length()); + + withTerminalSized(20, 4); + assertEnteringStringGivesResponse( + "\033_Ga=t,f=32,t=d,i=500,s=128,v=128,m=0;" + payload + "\033\\", + "\033_Gi=500;OK\033\\"); + assertEquals(128 * 128 * 4, mTerminal.getKittyImageDataLength(500)); + + // Nothing of the image data was printed on the terminal. + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testUnterminatedCommandWithLargeImageDataDoesNotPrintOnScreen() { + withTerminalSized(4, 4); + enterString("\033_Ga=t,f=100,t=d,i=15,m=0,q=2;"); + + // The command is never terminated with `ESC \`, so all of the data is received as the image + // data of the command and must not be printed as text. + StringBuilder payload = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + payload.append('A'); + } + for (int i = 0; i < 70; i++) { + enterString(payload.toString()); + } + + assertLinesAre(" ", " ", " ", " "); + assertEquals(-1, mTerminal.getKittyImageDataLength(15)); + assertEquals("", mOutput.getOutputAndClear()); + + // The terminal is not left in a broken state once the command is terminated. + enterString("\033\\"); + enterString("ok").assertLinesAre("ok ", " ", " ", " "); + } + + public void testCommandWithControlDataLongerThanTheMaxLengthIsDiscarded() { + withTerminalSized(4, 4); + + // The control data of a command is stored in the args buffer, so a command that never sends + // the `;` that ends it is discarded instead of being printed as text. + StringBuilder controlData = new StringBuilder("\033_G"); + for (int i = 0; i < 1000; i++) { + controlData.append("a=t,"); + } + enterString(controlData.toString()); + + assertLinesAre(" ", " ", " ", " "); + assertEquals("", mOutput.getOutputAndClear()); + + // The rest of the command is discarded as well, including a payload. + enterString(";AAAAAAAA\033\\"); + enterString("ok").assertLinesAre("ok ", " ", " ", " "); + } + + + /* Deleting images (`a=d`). */ + + public void testDeleteWithoutDeleteModeRespondsOk() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=16,m=0,q=2;" + PNG_BASE64 + "\033\\"); + + // The `d` key defaults to `d=a`, which only deletes placements and keeps the image data. + assertEnteringStringGivesResponse("\033_Ga=d,i=16\033\\", "\033_Gi=16;OK\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(16)); + + // This is what the herdr-browser plugin sends. + assertEnteringStringGivesResponse("\033_Ga=d,i=16,q=2\033\\", ""); + } + + /** + * Create a placement of a kitty graphics image directly instead of by sending an `a=T` or `a=p` + * command, since the android `Bitmap` class required to create one is not available in unit + * tests. The {@link TerminalBitmap} is created without a bitmap, which is enough to cover + * everything that operates on placements after they have been created. + * + * @param bitmapNum The bitmap number to create the placement with, which must not already be in + * use, so the placements must be created with `0`, `1`, `2`, ... in order. + * @param isKittyImage Whether the placement is of a kitty graphics image, so that a sixel or + * iTerm image can be created as well. + */ + private void addPlacement(int bitmapNum, boolean isKittyImage, long imageId, long placementId, + int row, int column, int columns) { + TerminalBuffer screen = mTerminal.getScreen(); + + TerminalBitmap terminalBitmap = new TerminalBitmap(null, bitmapNum, /* bitmap */ null, + INITIAL_CELL_WIDTH_PIXELS, INITIAL_CELL_HEIGHT_PIXELS, /* scrollLines */ 1, + new int[] {1, columns}); + if (isKittyImage) { + terminalBitmap.setKittyImage(imageId, placementId); + } + screen.addTerminalBitmap(terminalBitmap); + + for (int i = 0; i < columns; i++) { + screen.setChar(column + i, row, '+', TextStyle.encodeTerminalBitmap(bitmapNum, i, 0)); + } + } + + /** Whether a {@link TerminalBitmap} is still loaded for a bitmap number. */ + private boolean hasBitmap(int bitmapNum) { + return mTerminal.getScreen().getTerminalBitmap(TextStyle.encodeTerminalBitmap(bitmapNum, 0, 0)) != null; + } + + /** Whether the row still contains any {@link TerminalBitmap} cell. */ + private boolean hasBitmapInRow(int row) { + TerminalBuffer screen = mTerminal.getScreen(); + return screen.mLines[screen.externalToInternalRow(row)].mHasTerminalBitmap; + } + + public void testDeleteKittyImagePlacementsClearsTheCellsOfThePlacements() { + withTerminalSized(6, 4); + + addPlacement(/* bitmapNum */ 0, /* isKittyImage */ true, /* imageId */ 1, /* placementId */ 1, /* row */ 0, /* column */ 0, /* columns */ 2); + addPlacement(1, true, 1, 2, 1, 0, 2); + addPlacement(2, true, 2, 1, 2, 0, 2); + // A sixel or iTerm image, which a kitty graphics delete must never remove. + addPlacement(3, false, 0, 0, 3, 0, 2); + + assertLinesAre("++ ", "++ ", "++ ", "++ "); + assertTrue(hasBitmapInRow(0)); + + // Only the placement of the placement id passed with the `p` key is deleted. + mTerminal.getScreen().deleteKittyImagePlacements(1, 2); + assertLinesAre("++ ", " ", "++ ", "++ "); + assertTrue(hasBitmap(0)); + assertFalse(hasBitmap(1)); + assertTrue(hasBitmap(2)); + assertTrue(hasBitmap(3)); + assertTrue(hasBitmapInRow(0)); + assertFalse(hasBitmapInRow(1)); + + // All the placements of the image are deleted if the `p` key is not passed. + mTerminal.getScreen().deleteKittyImagePlacements(1, KittyImage.PLACEMENT_ID__NONE); + assertLinesAre(" ", " ", "++ ", "++ "); + assertFalse(hasBitmap(0)); + assertTrue(hasBitmap(2)); + assertTrue(hasBitmap(3)); + assertFalse(hasBitmapInRow(0)); + + // Deleting all the kitty graphics placements must not remove the sixel or iTerm image. + mTerminal.getScreen().deleteAllKittyImagePlacements(); + assertLinesAre(" ", " ", " ", "++ "); + assertFalse(hasBitmap(2)); + assertTrue(hasBitmap(3)); + assertTrue(hasBitmapInRow(3)); + } + + /* + * The cursor movement after an image. + * + * The android `Bitmap` class is not available in unit tests, so an image is never actually placed + * and `TerminalEmulator.moveCursorAfterTerminalBitmap()` is never reached through a command. + * It is called directly with the cursor delta a placement would have returned instead, which is + * the number of rows the image covers as the first value and the number of columns as the second. + */ + + public void testCursorAfterImageThatEndsExactlyAtTheLastColumn() { + // An image that ends exactly at the last column of the screen fits on the last row it covers, + // so the cursor must be left on that row at the column after the image, which is the column + // past the end of the screen. Note that the condition for this was `col < mColumns - 1` in the + // sixel and iTerm image code this is shared with, which treated such an image as not fitting. + withTerminalSized(56, 20); + + // A single row image in the columns 53 and 54 of a 56 column screen ends at the last column. + placeCursorAndAssert(2, 53); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {1, 2}); + assertCursorAt(2, 55); + + // A two row image ends on the row below the row it started on. + placeCursorAndAssert(8, 53); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {2, 2}); + assertCursorAt(9, 55); + } + + public void testCursorAfterImageThatDoesNotFitOnTheLastColumn() { + // An image that ends past the last column of the screen does not fit, so the cursor is wrapped + // to the first column of the row after the image. + withTerminalSized(56, 20); + + // A single row image in the columns 54 and 55 of a 56 column screen ends past the last column. + placeCursorAndAssert(2, 54); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {1, 2}); + assertCursorAt(3, 0); + + placeCursorAndAssert(8, 54); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {2, 2}); + assertCursorAt(10, 0); + } + + public void testCursorAfterImageThatEndsBeforeTheLastColumn() { + // The common case, where the cursor is left on the last row the image covers. + withTerminalSized(56, 20); + + placeCursorAndAssert(2, 0); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {1, 4}); + assertCursorAt(2, 4); + + placeCursorAndAssert(8, 10); + mTerminal.moveCursorAfterTerminalBitmap(new int[] {3, 4}); + assertCursorAt(10, 14); + } + + public void testUnreferencedTerminalBitmapsAreReleasedWhenTheirCellsAreOverwritten() { + // A placement that displays a new image at the position of a previous one overwrites its + // cells, which leaves the bitmap of the previous one unreferenced and it must be released + // without waiting for the throttled garbage collection, otherwise a client that streams a + // pane at frame rate keeps every frame in memory. + withTerminalSized(6, 3); + TerminalBuffer screen = mTerminal.getScreen(); + + addPlacement(/* bitmapNum */ 0, /* isKittyImage */ true, /* imageId */ 1, /* placementId */ 1, /* row */ 0, /* column */ 0, /* columns */ 2); + addPlacement(1, true, 1, 2, 1, 0, 2); + addPlacement(2, false, 0, 0, 2, 0, 2); + assertTrue(hasBitmap(0)); + assertTrue(hasBitmap(1)); + assertTrue(hasBitmap(2)); + + // Nothing is released while every bitmap is still referenced by a cell. + Set bitmapNums = new HashSet<>(Arrays.asList(0, 1, 2)); + screen.releaseUnreferencedTerminalBitmaps(bitmapNums); + assertTrue(hasBitmap(0)); + assertTrue(hasBitmap(1)); + assertTrue(hasBitmap(2)); + + // Overwrite the cells of the first placement, like a new placement at the same position does. + screen.setChar(0, 0, 'a', TextStyle.NORMAL); + screen.setChar(1, 0, 'b', TextStyle.NORMAL); + + screen.releaseUnreferencedTerminalBitmaps(new HashSet<>(Arrays.asList(0, 1, 2))); + assertFalse(hasBitmap(0)); + assertTrue(hasBitmap(1)); + assertTrue(hasBitmap(2)); + + // A `null` or empty set of candidates is a no operation. + screen.releaseUnreferencedTerminalBitmaps(null); + screen.releaseUnreferencedTerminalBitmaps(new HashSet()); + assertTrue(hasBitmap(1)); + assertTrue(hasBitmap(2)); + } + + public void testReferencedTerminalBitmapsInTheTranscriptAreNotReleased() { + // A bitmap that is only referenced by a row that has scrolled into the transcript must not be + // released, since it is still drawn when the transcript is scrolled back to. + withTerminalSized(6, 2); + addPlacement(0, true, 1, 1, 0, 0, 2); + assertTrue(hasBitmap(0)); + + // Scroll the placement off the screen and into the transcript. + enterString("\033[2;1H\r\n\r\n"); + assertTrue(mTerminal.getScreen().getActiveTranscriptRows() > 0); + + mTerminal.getScreen().releaseUnreferencedTerminalBitmaps(new HashSet<>(Arrays.asList(0))); + assertTrue(hasBitmap(0)); + } + + public void testDeleteAllKittyImagePlacementsDeletesPlacementsWithoutAnImageId() { + // A placement created by a command that did not pass the `i` key can only be deleted with a + // `d=a` or `d=A` delete, so it must not be treated as a placement of a non kitty image. + withTerminalSized(6, 2); + addPlacement(0, true, KittyImage.IMAGE_ID__NONE, KittyImage.PLACEMENT_ID__NONE, 0, 0, 2); + addPlacement(1, false, 0, 0, 1, 0, 2); + assertLinesAre("++ ", "++ "); + + mTerminal.getScreen().deleteAllKittyImagePlacements(); + assertLinesAre(" ", "++ "); + assertFalse(hasBitmap(0)); + assertTrue(hasBitmap(1)); + } + + public void testDeleteAllKittyImagePlacementsWithoutAnyPlacementDoesNothing() { + withTerminalSized(6, 2); + addPlacement(0, false, 0, 0, 0, 0, 2); + assertLinesAre("++ ", " "); + + mTerminal.getScreen().deleteAllKittyImagePlacements(); + mTerminal.getScreen().deleteKittyImagePlacements(1, 1); + assertLinesAre("++ ", " "); + assertTrue(hasBitmap(0)); + } + + public void testDeleteByIdKeepsOrFreesImageData() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=17,m=0,q=2;" + PNG_BASE64 + "\033\\"); + enterString("\033_Ga=t,f=100,t=d,i=18,m=0,q=2;" + PNG_BASE64 + "\033\\"); + + // The lowercase `d=i` only deletes the placements of the image and keeps its data. + assertEnteringStringGivesResponse("\033_Ga=d,d=i,i=17\033\\", "\033_Gi=17;OK\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(17)); + + // The uppercase `d=I` also frees the image data, and only for the image id passed. + assertEnteringStringGivesResponse("\033_Ga=d,d=I,i=17\033\\", "\033_Gi=17;OK\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(17)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(18)); + } + + public void testDeleteAllFreesImageDataOnlyForUppercaseDeleteMode() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=19,m=0,q=2;" + PNG_BASE64 + "\033\\"); + enterString("\033_Ga=t,f=100,t=d,i=20,m=0,q=2;" + PNG_BASE64 + "\033\\"); + + assertEnteringStringGivesResponse("\033_Ga=d,d=a,i=19\033\\", "\033_Gi=19;OK\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(19)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(20)); + + assertEnteringStringGivesResponse("\033_Ga=d,d=A,i=19\033\\", "\033_Gi=19;OK\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(19)); + assertEquals(-1, mTerminal.getKittyImageDataLength(20)); + } + + public void testDeleteOfUnknownImageIdRespondsOk() { + withTerminalSized(20, 4); + + // Deleting an image that does not exist is not an error as per the protocol. + assertEnteringStringGivesResponse("\033_Ga=d,d=i,i=21\033\\", "\033_Gi=21;OK\033\\"); + assertEnteringStringGivesResponse("\033_Ga=d,d=I,i=21\033\\", "\033_Gi=21;OK\033\\"); + assertEnteringStringGivesResponse("\033_Ga=d,d=i,i=21,q=2\033\\", ""); + + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testDeleteByIdWithoutImageIdRespondsError() { + withTerminalSized(20, 4); + // No response can be sent without an image id, so only check that nothing breaks. + assertEnteringStringGivesResponse("\033_Ga=d,d=i\033\\", ""); + enterString("ok").assertLinesAre("ok ", " ", + " ", " "); + } + + public void testUnsupportedDeleteModeRespondsError() { + withTerminalSized(20, 4); + + // The delete modes that require image numbers, placement ids or coordinates. + assertEnteringStringGivesResponse("\033_Ga=d,d=n,i=22\033\\", + "\033_Gi=22;ENOTSUP:unsupported delete mode\033\\"); + assertEnteringStringGivesResponse("\033_Ga=d,d=z,i=22\033\\", + "\033_Gi=22;ENOTSUP:unsupported delete mode\033\\"); + } + + + /* The command sequences of actual clients. */ + + public void testHerdrCommandSequence() { + // The exact sequences the herdr client sends, which transmits an image once and then + // displays it again for every frame with an `a=p` command. Every command passes `q=2`, so the + // terminal must stay completely silent, and the cursor is positioned with `CUP` before every + // placement, which `C=1` requires to be left alone. + withTerminalSized(20, 6); + + // Transmit, in two chunks like herdr does for image data larger than one chunk. + enterString("\033_Ga=t,t=d,f=100,s=1,v=1,i=424242,q=2,m=1;" + PNG_BASE64_CHUNK_0 + "\033\\"); + enterString("\033_Gm=0,q=2;" + PNG_BASE64_CHUNK_1 + "\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(424242)); + + // Place, for two consecutive frames without transmitting the image again. + enterString("\033[2;1H"); + enterString("\033_Ga=p,i=424242,p=1,c=20,r=4,z=0,C=1,q=2\033\\"); + assertCursorAt(1, 0); + enterString("\033_Ga=d,d=i,i=424242,p=1,q=2;\033\\"); + enterString("\033[2;1H"); + // The optional keys herdr only adds when the corresponding fields are non zero. The source + // rectangle covers the entire `1x1` image that was transmitted with `s=1,v=1` above. + enterString("\033_Ga=p,i=424242,p=1,c=20,r=4,z=0,C=1,q=2,x=0,y=0,w=1,h=1,X=0,Y=0\033\\"); + assertCursorAt(1, 0); + + // The image data is kept for every frame until it is explicitly freed. + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(424242)); + enterString("\033_Ga=d,d=I,i=424242,q=2;\033\\"); + assertEquals(-1, mTerminal.getKittyImageDataLength(424242)); + + assertEquals("", mOutput.getOutputAndClear()); + assertLinesAre(" ", " ", " ", + " ", " ", " "); + } + + public void testYaziCapabilityQueryAndRawTransmitSequence() { + // Yazi probes with a capability query followed by `CSI c` (Primary Device Attributes), and + // expects the graphics response before the device attributes response if the protocol is + // supported. It transmits raw `RGB`/`RGBA` pixel data and never `PNG`. + withTerminalSized(20, 6); + assertEnteringStringGivesResponse("\033_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\033\\\033[c", + "\033_Gi=31;OK\033\\" + "\033[?64;1;2;4;6;9;15;18;21;22c"); + + assertEnteringStringGivesResponse( + "\033_Ga=T,f=32,t=d,i=31,s=2,v=2,c=2,r=1,m=0,q=1;" + RGBA_BASE64_2X2 + "\033\\", + // Creating the bitmap always fails in unit tests, and `q=1` does not suppress errors. + "\033_Gi=31;EBADPNG:displaying image failed\033\\"); + assertEquals(16, mTerminal.getKittyImageDataLength(31)); + } + + + /* Regressions for other APC commands. */ + + public void testNonKittyApcCommandsAreStillIgnoredSilently() { + // Only `APC` commands starting with `G` are kitty graphics commands. + withTerminalSized(12, 2); + assertEnteringStringGivesResponse("hello \033_some\023\033_\\apc#end\033\\ world", ""); + assertLinesAre("hello world", " "); + + withTerminalSized(12, 2); + assertEnteringStringGivesResponse("\033_Ha=T,f=100,t=d,i=1,m=0;" + PNG_BASE64 + "\033\\", ""); + assertLinesAre(" ", " "); + + withTerminalSized(12, 2); + assertEnteringStringGivesResponse("\033_\033\\", ""); + assertLinesAre(" ", " "); + } + + public void testOldestKittyImageDataIsEvictedWhenTooManyImagesAreTransmitted() { + withTerminalSized(20, 4); + + // At most 32 images are stored, so transmitting 33 images must evict the oldest one. + for (int imageId = 1; imageId <= 33; imageId++) { + enterString("\033_Ga=t,f=100,t=d,i=" + imageId + ",m=0,q=2;" + PNG_BASE64 + "\033\\"); + } + + assertEquals(-1, mTerminal.getKittyImageDataLength(1)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(2)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(33)); + + // Re-transmitting an image does not evict any other image. + enterString("\033_Ga=t,f=100,t=d,i=33,m=0,q=2;" + PNG_BASE64 + "\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(2)); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(33)); + } + + public void testKittyImageDataIsFreedOnTerminalReset() { + withTerminalSized(20, 4); + enterString("\033_Ga=t,f=100,t=d,i=23,m=0,q=2;" + PNG_BASE64 + "\033\\"); + assertEquals(PNG_LENGTH, mTerminal.getKittyImageDataLength(23)); + + // `ESC c` - full reset (RIS). + enterString("\033c"); + assertEquals(-1, mTerminal.getKittyImageDataLength(23)); + } + +}