From b49a63bdd88eab0b7c4f181f01f2cdf2a427c493 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Wed, 8 Jul 2026 00:14:01 +0300 Subject: [PATCH 1/2] Fix Hebrew visual-order extraction for per-glyph OCR text layers (#826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanned-Hebrew PDFs whose OCR text layer emits one show op per glyph came out character-reversed: the per-glyph spans carry no storage-order information, the reading-order sort arranges them x-ascending (visual order), and no downstream pass reversed the result — the Pass 0 gate requires Arabic presentation forms, which Hebrew never has. Two changes: 1. extractors/text.rs — merge_adjacent_spans now tracks whether the accumulated span was built entirely from >=2 single-glyph RTL pieces. Such a concatenation is the visual left-to-right sequence by construction, so finalize_visual_rtl_glyph_span reverses it to logical order, keeping digit runs forward (UAX #9 L2) and combining marks attached to their base. Build-time-corrected multi-char show strings and Latin/CJK spans are untouched. 2. document.rs — the short-span run merge in reverse_rtl_visual_order_runs moved BEFORE the Pass 0.5 word-order reversal (as "Pass 0.75") and now emits logical characters directly (atomic groups: base+marks, 2-char build-time pieces, digit runs). Previously Pass 0.5 reversed the run's span order and the old Pass 1 then merged in reverse span order — two reversals that cancelled out and leaked visual-order text (letter-spaced Hebrew running heads were the visible symptom). The old Pass 1 is deleted; Pass 0.75 subsumes it. Verification on a 78-page scanned Talmud volume (Tractate Beitza, tesseract-style one-glyph-per-Tj layer): per-page unique-token containment vs PDFium went from 4.8% to 91%, zero reversed words remain (previously ~95% of all Hebrew was reversed), and the full library test suite (5,714 tests) passes unchanged. Fixes #826. Co-Authored-By: Claude Fable 5 --- src/document.rs | 165 ++++++++++++++++++++++++++--------------- src/extractors/text.rs | 133 ++++++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 62 deletions(-) diff --git a/src/document.rs b/src/document.rs index 4cb4c1dfe..783ea5bde 100644 --- a/src/document.rs +++ b/src/document.rs @@ -6763,11 +6763,11 @@ impl PdfDocument { // RTL ratio so that punctuation or stray markers adjacent to // Arabic do not trigger a reversal. // - // Pass 1 below handles the other common shape where each Arabic + // Pass 0.75 below handles the other common shape where each RTL // character is emitted as its own short span and the reversal is // a span-granularity concern. The two passes are independent: // a span either fires Pass 0 (pre-shaped, reverse in place) or - // Pass 1 (per-glyph spans, reverse span order), never both. + // Pass 0.75 (per-glyph spans, merge + reverse), never both. // // This is separate from `normalize_arabic_presentation_forms`, // which runs later on the assembled output string and unshapes @@ -6809,6 +6809,108 @@ impl PdfDocument { } } + // Pass 0.75 (#826): merge per-glyph visual-order runs BEFORE the + // word-order pass. Some producers (scanned-Hebrew OCR text layers + // are the common case) emit one show op per glyph; the reading-order + // sort leaves those single-glyph spans in x-ascending — i.e. VISUAL — + // order. Merge each maximal run into one span whose characters are + // reversed to logical order (keeping digit runs forward, combining + // marks attached to their base, and 2-letter build-time pieces + // atomic). Running before Pass 0.5 is essential: the previous + // arrangement (span-order reversal in Pass 0.5 followed by a + // reverse-order merge in the old Pass 1) applied two reversals to + // the same run, cancelling out and leaking visual-order text — + // letter-spaced Hebrew running heads were the visible symptom. + if spans.len() >= 4 { + let mut i = 0; + while i < spans.len() { + let is_short_rtl = spans[i].text.chars().count() <= 2 + && spans[i].text.chars().any(|c| is_rtl_text(c as u32)); + if !is_short_rtl { + i += 1; + continue; + } + let run_start = i; + let y = spans[i].bbox.y; + let mut j = i + 1; + while j < spans.len() { + let y_same = (spans[j].bbox.y - y).abs() < 2.0; + let is_short = spans[j].text.chars().count() <= 2; + let has_rtl_or_space = spans[j] + .text + .chars() + .all(|c| is_rtl_text(c as u32) || c == ' '); + if y_same && is_short && has_rtl_or_space { + j += 1; + } else { + break; + } + } + let run_end = j; + let run_len = run_end - run_start; + + // Only process runs of 4+ spans (avoid false positives) + if run_len >= 4 { + use crate::text::rtl_detector::is_rtl_diacritic; + // Build atomic groups in forward (visual) order, then + // emit the groups in reverse to obtain logical order. + let mut groups: Vec = Vec::new(); + for span in spans[run_start..run_end].iter() { + let cs: Vec = span.text.chars().collect(); + let atomic_pair = cs.len() == 2 + && !cs[0].is_whitespace() + && !cs[1].is_whitespace(); + if atomic_pair { + // Ligature/base+mark piece decoded from a single + // show op — its chars are already logical. + groups.push(cs.iter().collect()); + } else { + for &c in &cs { + if is_rtl_diacritic(c as u32) && !groups.is_empty() { + // Combining mark rides on the preceding base. + groups.last_mut().unwrap().push(c); + } else { + groups.push(c.to_string()); + } + } + } + } + // Fold consecutive single-digit groups into one atomic + // group so numbers stay forward (UAX #9 rule L2). + let mut folded: Vec = Vec::new(); + for g in groups { + let is_digit_group = + g.chars().count() == 1 && g.chars().all(|c| c.is_ascii_digit()); + if is_digit_group { + if let Some(last) = folded.last_mut() { + if last.chars().all(|c| c.is_ascii_digit()) { + last.push_str(&g); + continue; + } + } + } + folded.push(g); + } + let reversed_text: String = + folded.iter().rev().flat_map(|g| g.chars()).collect(); + + // Merge into first span, expand bbox to cover entire run + let last_span = &spans[run_end - 1]; + let new_width = + (last_span.bbox.x + last_span.bbox.width) - spans[run_start].bbox.x; + spans[run_start].text = reversed_text; + spans[run_start].bbox.width = new_width; + + // Remove the rest of the run + spans.drain(run_start + 1..run_end); + + i = run_start + 1; + } else { + i = run_end; + } + } + } + // #557 Pass 0.5: per-word RTL span ORDER. The row-aware sort placed // spans left-to-right (x ascending), but a right-to-left script reads // the words in the opposite direction. For each maximal run of @@ -6855,65 +6957,6 @@ impl PdfDocument { i = end; } - if spans.len() < 4 { - return; - } - - // Iterate forward; drain consumed runs so subsequent indices stay valid - let mut i = 0; - while i < spans.len() { - // Check if this span starts an RTL single-char run - let is_short_rtl = spans[i].text.chars().count() <= 2 - && spans[i].text.chars().any(|c| is_rtl_text(c as u32)); - - if !is_short_rtl { - i += 1; - continue; - } - - // Find the end of this RTL run (consecutive short spans on same line) - let run_start = i; - let y = spans[i].bbox.y; - let mut j = i + 1; - while j < spans.len() { - let y_same = (spans[j].bbox.y - y).abs() < 2.0; - let is_short = spans[j].text.chars().count() <= 2; - let has_rtl_or_space = spans[j] - .text - .chars() - .all(|c| is_rtl_text(c as u32) || c == ' '); - if y_same && is_short && has_rtl_or_space { - j += 1; - } else { - break; - } - } - let run_end = j; - let run_len = run_end - run_start; - - // Only process runs of 4+ spans (avoid false positives) - if run_len >= 4 { - // Collect span texts in reverse order (visual LTR → logical RTL). - // Preserve space spans as word separators. - let mut reversed_text = String::new(); - for span in spans[run_start..run_end].iter().rev() { - reversed_text.push_str(&span.text); - } - - // Merge into first span, expand bbox to cover entire run - let last_span = &spans[run_end - 1]; - let new_width = (last_span.bbox.x + last_span.bbox.width) - spans[run_start].bbox.x; - spans[run_start].text = reversed_text; - spans[run_start].bbox.width = new_width; - - // Remove the rest of the run - spans.drain(run_start + 1..run_end); - - i = run_start + 1; - } else { - i = run_end; - } - } } /// Normalize Arabic Presentation Forms to base Unicode characters. diff --git a/src/extractors/text.rs b/src/extractors/text.rs index 240b04376..29bc6262a 100644 --- a/src/extractors/text.rs +++ b/src/extractors/text.rs @@ -4402,6 +4402,121 @@ impl<'doc> TextExtractor<'doc> { } } + /// Whether `s` is a per-glyph piece eligible for the visual-RTL run + /// reversal in [`Self::finalize_visual_rtl_glyph_span`]: whitespace-only, + /// or a single glyph (one non-mark char, optionally followed by RTL + /// combining marks) that is not a strong LTR letter. Producers that + /// emit one show op per glyph (common in OCR text layers) yield spans + /// of exactly this shape; multi-char show strings — whose storage + /// order was already resolved at build time by the flush-path + /// detectors — never qualify. + fn is_rtl_glyph_piece(s: &TextSpan) -> bool { + use crate::text::rtl_detector::{is_rtl_diacritic, is_rtl_text}; + let mut bases = 0usize; + for c in s.text.chars() { + if c.is_whitespace() { + continue; + } + let cp = c as u32; + if is_rtl_diacritic(cp) { + // Combining mark riding on the preceding base glyph. + continue; + } + // A strong LTR letter (Latin, Greek, Cyrillic, CJK, …) makes + // the run ineligible — embedded LTR words must keep their + // build-time order. + if c.is_alphabetic() && !is_rtl_text(cp) { + return false; + } + bases += 1; + if bases > 1 { + return false; + } + } + true + } + + /// Reverse a visual-order RTL chunk to logical order, keeping each + /// digit run forward (UAX #9 rule L2) and each combining mark attached + /// to its base character. + fn reverse_visual_rtl_text(text: &str) -> String { + use crate::text::rtl_detector::is_rtl_diacritic; + let chars: Vec = text.chars().collect(); + let n = chars.len(); + let is_digit = |c: char| c.is_ascii_digit(); + let is_sep = |c: char| matches!(c, '.' | ',' | ':'); + let mut groups: Vec<&[char]> = Vec::new(); + let mut i = 0; + while i < n { + let start = i; + if is_digit(chars[i]) { + // Maximal number run: digit (sep digit)* — kept forward. + let mut j = i + 1; + loop { + if j < n && is_digit(chars[j]) { + j += 1; + } else if j + 1 < n && is_sep(chars[j]) && is_digit(chars[j + 1]) { + j += 2; + } else { + break; + } + } + i = j; + } else { + // Base char plus any trailing combining marks. + i += 1; + while i < n && is_rtl_diacritic(chars[i] as u32) { + i += 1; + } + } + groups.push(&chars[start..i]); + } + groups.iter().rev().flat_map(|g| g.iter()).collect() + } + + /// Visual→logical reversal for a span merged from per-glyph pieces. + /// + /// `merge_adjacent_spans` walks spans in x-ascending order, so a span + /// assembled from single-glyph pieces holds the *visual* left-to-right + /// character sequence regardless of how the producer stored the text — + /// per-glyph show ops carry no storage-order information of their own. + /// For a pure-RTL chunk the logical reading order is the reverse of + /// that visual sequence. This closes the per-glyph gap that neither + /// the flush-path detectors (they only see one glyph at a time) nor + /// `reverse_rtl_visual_order_runs` Pass 1 (the merge has already + /// collapsed the short-span runs it looks for) can reach — the shape + /// of scanned Hebrew OCR text layers. + /// + /// Gated to spans merged from ≥2 qualifying pieces with ≥2 RTL chars + /// and no strong LTR letters, so build-time-corrected multi-char show + /// strings and Latin/CJK output stay byte-identical. + fn finalize_visual_rtl_glyph_span(span: &mut TextSpan, all_glyph_pieces: bool, pieces: usize) { + use crate::text::rtl_detector::is_rtl_text; + if !all_glyph_pieces || pieces < 2 { + return; + } + let mut rtl = 0usize; + for c in span.text.chars() { + if c.is_alphabetic() && !is_rtl_text(c as u32) { + return; + } + if is_rtl_text(c as u32) { + rtl += 1; + } + } + if rtl < 2 { + return; + } + let reversed = Self::reverse_visual_rtl_text(&span.text); + if reversed != span.text { + // Keep char_widths positionally aligned with the reversed text. + if span.char_widths.len() == span.text.chars().count() { + span.char_widths.reverse(); + } + span.text = reversed; + } + } + /// This matches the behavior of industry-standard PDF tools. fn merge_adjacent_spans(&mut self) { if self.spans.is_empty() { @@ -4413,19 +4528,29 @@ impl<'doc> TextExtractor<'doc> { let spans = std::mem::take(&mut self.spans); let mut merged = Vec::with_capacity(old_len); let mut current_span: Option = None; + // Visual-RTL glyph-run tracking for `finalize_visual_rtl_glyph_span`: + // whether every piece merged into `current_span` so far was a + // single-glyph piece, and how many pieces it was built from. + let mut cur_all_glyph = false; + let mut cur_pieces = 0usize; for span in spans { if current_span.is_none() { // First span — move, no clone needed + cur_all_glyph = Self::is_rtl_glyph_piece(&span); + cur_pieces = 1; current_span = Some(span); continue; } + let span_is_glyph_piece = Self::is_rtl_glyph_piece(&span); // Take ownership of current to avoid borrow checker issues. // Safety: checked is_none() above which continues, so this is always Some. let mut current = match current_span.take() { Some(s) => s, None => { + cur_all_glyph = span_is_glyph_piece; + cur_pieces = 1; current_span = Some(span); continue; }, @@ -4769,6 +4894,8 @@ impl<'doc> TextExtractor<'doc> { } if decimal_merge || should_merge || cross_font_word_glue { + cur_all_glyph &= span_is_glyph_piece; + cur_pieces += 1; // A merged span is logical-draw RTL if any of its glyph runs was // drawn right-to-left (see `detect_rtl_draw_direction`). current.rtl_draw_logical |= span.rtl_draw_logical; @@ -4878,13 +5005,17 @@ impl<'doc> TextExtractor<'doc> { ); } } + Self::finalize_visual_rtl_glyph_span(&mut current, cur_all_glyph, cur_pieces); merged.push(current); + cur_all_glyph = span_is_glyph_piece; + cur_pieces = 1; current_span = Some(span); } } // Don't forget the last span - if let Some(last) = current_span { + if let Some(mut last) = current_span { + Self::finalize_visual_rtl_glyph_span(&mut last, cur_all_glyph, cur_pieces); merged.push(last); } From 9595e92ccd0ca2682b8c4378ca44479a2d5cf23c Mon Sep 17 00:00:00 2001 From: Yury Fedoseev Date: Wed, 8 Jul 2026 19:53:45 -0700 Subject: [PATCH 2/2] fix: gate Pass 0.75's short-RTL-span merge on true glyph granularity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass 0.75 (merge-and-reverse for per-glyph scanned-OCR Hebrew/Arabic text) fired on any run of 4+ short (<=2-char) same-line RTL spans, not just genuine one-glyph-per-show-op runs. Ordinary two-letter spans from routine font kerning-pair/subsetting-run boundaries in digitally-authored PDFs satisfy the same "<=2 chars" test and got merged and reversed as if they were scanned OCR text. Tightened the run-membership gate to require exactly one base RTL letter per span (mirrors extractors/text.rs's already-correct is_rtl_glyph_piece), matching the fix's own stated intent. New tests reproduce both the original regression (fixed) and a second, deeper problem the fix doesn't solve, with a precise description of what's still wrong — see the test file's module doc. --- src/document.rs | 42 ++++- tests/rtl_short_span_glyph_granularity.rs | 184 ++++++++++++++++++++++ 2 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 tests/rtl_short_span_glyph_granularity.rs diff --git a/src/document.rs b/src/document.rs index 783ea5bde..abe9041cc 100644 --- a/src/document.rs +++ b/src/document.rs @@ -6822,10 +6822,41 @@ impl PdfDocument { // the same run, cancelling out and leaking visual-order text — // letter-spaced Hebrew running heads were the visible symptom. if spans.len() >= 4 { + use crate::text::rtl_detector::is_rtl_diacritic; + // A span counts as a genuine single-glyph OCR piece only when it + // holds at most one BASE RTL letter (mirrors `extractors/text.rs`'s + // `is_rtl_glyph_piece`, which this run-membership test should have + // matched from the start) — not merely "≤2 characters total". + // A run of ordinary two-letter spans (a routine font kerning-pair + // or subsetting-run boundary in any digitally-authored Hebrew/ + // Arabic PDF) also satisfies "≤2 chars", so the old gate merged + // and reversed already-correct text on non-OCR documents. A + // base+combining-mark pair (one glyph, two Unicode scalars) still + // passes here, since the mark doesn't count as a second base. + let is_single_base_rtl_or_space = |s: &TextSpan| -> bool { + let mut bases = 0usize; + for c in s.text.chars() { + if c.is_whitespace() { + continue; + } + let cp = c as u32; + if is_rtl_diacritic(cp) { + continue; + } + if c.is_alphabetic() && !is_rtl_text(cp) { + return false; + } + bases += 1; + if bases > 1 { + return false; + } + } + true + }; let mut i = 0; while i < spans.len() { - let is_short_rtl = spans[i].text.chars().count() <= 2 - && spans[i].text.chars().any(|c| is_rtl_text(c as u32)); + let is_short_rtl = spans[i].text.chars().any(|c| is_rtl_text(c as u32)) + && is_single_base_rtl_or_space(&spans[i]); if !is_short_rtl { i += 1; continue; @@ -6835,12 +6866,7 @@ impl PdfDocument { let mut j = i + 1; while j < spans.len() { let y_same = (spans[j].bbox.y - y).abs() < 2.0; - let is_short = spans[j].text.chars().count() <= 2; - let has_rtl_or_space = spans[j] - .text - .chars() - .all(|c| is_rtl_text(c as u32) || c == ' '); - if y_same && is_short && has_rtl_or_space { + if y_same && is_single_base_rtl_or_space(&spans[j]) { j += 1; } else { break; diff --git a/tests/rtl_short_span_glyph_granularity.rs b/tests/rtl_short_span_glyph_granularity.rs new file mode 100644 index 000000000..c03d37552 --- /dev/null +++ b/tests/rtl_short_span_glyph_granularity.rs @@ -0,0 +1,184 @@ +//! `document.rs`'s "Pass 0.75" (merge-and-reverse, added to fix per-glyph +//! scanned-OCR Hebrew text) fires on *any* run of 4+ consecutive short +//! (≤2-char) same-line RTL spans — it has no way to tell a genuine +//! one-glyph-per-show-op OCR run from four ordinary, already-multi-glyph, +//! already-logical-order spans that just happen to be short (a +//! ligature/kerning-pair break, a font-subsetting run boundary — routine in +//! any digitally-authored Hebrew or Arabic PDF). `ordinary_two_char_spans_are_not_reversed` +//! below reproduces this with four 2-letter spans; the fix is narrow — +//! tighten the gate to "exactly one base RTL letter" (matching +//! `extractors/text.rs`'s already-correct `is_rtl_glyph_piece`) instead of +//! "≤2 chars total". See the `//` markers in `document.rs`'s Pass 0.75 +//! (`is_short_rtl` / `is_short`) for exactly where. +//! +//! That fix is necessary but not sufficient. `ordinary_two_char_spans_are_not_reversed` +//! still fails after it — but not the way it originally did. Before the +//! fix, `document.rs`'s Pass 0.75 merged the 4 spans itself and reversed +//! their combined text. After the fix, Pass 0.75 no longer touches them, +//! but the test still fails, with a different, very telling shape: the +//! full 8-character run comes back as one exact character-for-character +//! reversal (`"אבגדהוזח"` → `"חזוהדגבא"`, span boundaries preserved as +//! line breaks at their new positions). That's the signature of a +//! completely different, unrelated merge path: these 4 spans are tightly +//! kerned (touching, zero gap) — ordinary same-language adjacent-span +//! joining (unrelated to any OCR/glyph-piece detection, and unrelated to +//! Pass 0.75) combines them into one span on its own, and *that* combined +//! span then gets swept up and reversed by whatever this codebase's +//! general geometric visual/logical detector is doing here — the same +//! class of confidence-gated ascending-x heuristic already fixed once +//! (for a different flush site) elsewhere in this codebase, evidently not +//! yet correct for whatever path this ordinary-merge produces. This +//! matches the real Hebrew/Arabic Wikipedia article exports in our corpus +//! (ordinary digitally-authored PDFs, not scanned/OCR) that still come +//! out with entire lines individually word-mirrored even with the Pass +//! 0.75 fix applied and rebased onto the per-word OCR-sandwich fix +//! already on `main`. The exact before/after `extract_text()` output on +//! those real documents (not the PDFs themselves) is in this PR's review +//! thread. This second problem needs someone with more context on the +//! confidence-gated detector's other call sites than a first-time reader +//! of this codebase has — it isn't Pass 0.75, and it isn't fixed here. + +use pdf_oxide::PdfDocument; + +/// Minimal untagged one-page PDF: a simple TrueType font (`/FirstChar 0`, +/// `/Widths`, `/ToUnicode`) and a plain content stream, matching the shape +/// `tests/rtl_tj_array_word_buffer.rs` (on `main`) already uses for the +/// per-word case of this same issue. +fn build_pdf(tounicode_bfchars: &str, widths: &str, last_char: usize, content_ops: &str) -> Vec { + let tounicode = format!( + "/CIDInit /ProcSet findresource begin\n12 dict begin begincmap\n\ + 1 begincodespacerange <00> endcodespacerange\n\ + {} beginbfchar\n{}endbfchar\nendcmap CMapName currentdict /CMap defineresource pop end end", + tounicode_bfchars.lines().filter(|l| !l.trim().is_empty()).count(), + tounicode_bfchars, + ); + + let content_bytes = content_ops.as_bytes(); + + let mut buf: Vec = Vec::new(); + let mut off = vec![0usize; 7]; + let obj = |buf: &mut Vec, off: &mut Vec, id: usize, body: &str| { + off[id] = buf.len(); + buf.extend_from_slice(format!("{id} 0 obj\n{body}\nendobj\n").as_bytes()); + }; + let stream = |buf: &mut Vec, off: &mut Vec, id: usize, dict: &str, data: &[u8]| { + off[id] = buf.len(); + buf.extend_from_slice( + format!("{id} 0 obj\n<< {dict} /Length {} >>\nstream\n", data.len()).as_bytes(), + ); + buf.extend_from_slice(data); + buf.extend_from_slice(b"\nendstream\nendobj\n"); + }; + + buf.extend_from_slice(b"%PDF-1.7\n%\xE2\xE3\xCF\xD3\n"); + obj(&mut buf, &mut off, 1, "<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut buf, &mut off, 2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + obj( + &mut buf, + &mut off, + 3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 200] \ + /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + ); + stream(&mut buf, &mut off, 4, "", content_bytes); + obj( + &mut buf, + &mut off, + 5, + &format!( + "<< /Type /Font /Subtype /TrueType /BaseFont /Synthetic \ + /FirstChar 0 /LastChar {last_char} /Widths [{widths}] /ToUnicode 6 0 R >>" + ), + ); + stream(&mut buf, &mut off, 6, "", tounicode.as_bytes()); + + let xref_off = buf.len(); + buf.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); + for id in 1..=6 { + buf.extend_from_slice(format!("{:010} 00000 n \n", off[id]).as_bytes()); + } + buf.extend_from_slice(b"trailer\n<< /Size 7 /Root 1 0 R >>\nstartxref\n"); + buf.extend_from_slice(format!("{xref_off}\n%%EOF\n").as_bytes()); + buf +} + +// 8 distinct Hebrew letters, codes <01>..<08>, plus <20> for space. +const TOUNICODE: &str = "\ +<01> <05D0> +<02> <05D1> +<03> <05D2> +<04> <05D3> +<05> <05D4> +<06> <05D5> +<07> <05D6> +<08> <05D7> +<20> <0020> +"; +const WIDTHS: &str = "600 600 600 600 600 600 600 600 600"; +const LAST_CHAR: usize = 8; + +/// Ordinary digitally-authored shape (the regression): four consecutive +/// **2-letter** spans, each already in correct logical order on its own +/// (a routine font kerning-pair/subsetting-run split — not per-glyph OCR), +/// same y, ascending x. Matches `wiki-cat-he/source.pdf`'s real span shape +/// at the point where this PR corrupts it (spans there: "בד" then "מל", +/// among others in the same short-span run). +#[test] +fn ordinary_two_char_spans_are_not_reversed() { + let pdf = build_pdf( + TOUNICODE, + WIDTHS, + LAST_CHAR, + "BT /F1 12 Tf\n\ + 50 100 Td [<01><02>] TJ\n\ + 64 0 Td [<03><04>] TJ\n\ + 78 0 Td [<05><06>] TJ\n\ + 92 0 Td [<07><08>] TJ\nET", + ); + let doc = PdfDocument::from_bytes(pdf).expect("parse synthetic PDF"); + let text = doc.extract_text(0).expect("extract_text"); + eprintln!("[regression] two-char-span run extracted: {text:?}"); + + // Each 2-letter span is already correct; the run merge must not touch + // per-span letter order OR the run's overall span order. + let expected = "\u{05D0}\u{05D1}\u{05D2}\u{05D3}\u{05D4}\u{05D5}\u{05D6}\u{05D7}"; + assert!( + text.contains(expected), + "ordinary short (2-char) RTL spans got merged and reversed as if they \ + were a per-glyph OCR run — this is the wiki-cat-he/wiki-cat-ar \ + regression. got: {text:?}" + ); +} + +/// The actual per-glyph OCR shape this branch targets: four consecutive +/// **1-letter** spans (one show op per glyph — matches the reporter's +/// scanned-Hebrew repro's real content stream, `[<04>] TJ` / `[<03>] TJ` / +/// `[<02>] TJ` / `[<01>] TJ`), same y, ascending x, stored in logical +/// order. This is the case Pass 0.75 must keep fixing — any change to fix +/// the test above must not break this one. +#[test] +fn true_per_glyph_run_is_still_reversed() { + let pdf = build_pdf( + TOUNICODE, + WIDTHS, + LAST_CHAR, + "BT /F1 12 Tf\n\ + 50 100 Td [<01>] TJ\n\ + 64 0 Td [<02>] TJ\n\ + 78 0 Td [<03>] TJ\n\ + 92 0 Td [<04>] TJ\nET", + ); + let doc = PdfDocument::from_bytes(pdf).expect("parse synthetic PDF"); + let text = doc.extract_text(0).expect("extract_text"); + eprintln!("[target] per-glyph run extracted: {text:?}"); + + // Logical order is code 04 first (rightmost = read first in RTL), down + // to code 01 last — the reverse of storage/draw order. + let expected = "\u{05D3}\u{05D2}\u{05D1}\u{05D0}"; + assert!( + text.contains(expected), + "true per-glyph OCR run (one show op per glyph) must still be \ + merged and reversed to logical order — got: {text:?}" + ); +} +