diff --git a/.gitignore b/.gitignore
index f651cd4de..fd147b20b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -455,3 +455,10 @@ r/*.tar.gz
/elixir/deps/
/elixir/priv/
/elixir/mix.lock
+
+# Viewer build artifacts (never commit generated output)
+viewer/dotnet/**/bin/
+viewer/dotnet/**/obj/
+viewer/dist-browser/
+viewer/**/node_modules/
+viewer/**/.gradle/
diff --git a/cpp/include/pdf_oxide/pdf_oxide.hpp b/cpp/include/pdf_oxide/pdf_oxide.hpp
index 96f00a733..5e44dca21 100644
--- a/cpp/include/pdf_oxide/pdf_oxide.hpp
+++ b/cpp/include/pdf_oxide/pdf_oxide.hpp
@@ -176,6 +176,11 @@ struct Element {
std::string type;
std::string text;
Bbox rect;
+ /// ISO 32000-1 §9.10.2 mapping-provenance label ("to_unicode"/"encoding"/
+ /// "predefined_cmap"/"embedded_cmap"/"actual_text"/"fallback"), or empty
+ /// when unknown. "fallback" means the text is a fabricated glyph-index echo,
+ /// not read from the file.
+ std::string provenance;
};
/// A single interactive AcroForm field (PHASE-8 forms).
@@ -899,6 +904,11 @@ class Document {
code = 0;
e.text = detail::take_string(pdf_oxide_element_get_text(list, i, &code),
code, "Document::page_get_elements");
+ code = 0;
+ if (char *p = pdf_oxide_element_get_provenance(list, i, &code)) {
+ e.provenance = detail::take_string(p, code,
+ "Document::page_get_elements");
+ }
Bbox b{0, 0, 0, 0};
pdf_oxide_element_get_rect(list, i, &b.x, &b.y, &b.width, &b.height,
&code);
diff --git a/csharp/PdfOxide/Internal/NativeMethods.cs b/csharp/PdfOxide/Internal/NativeMethods.cs
index 88a20f0f8..d6714d3e4 100644
--- a/csharp/PdfOxide/Internal/NativeMethods.cs
+++ b/csharp/PdfOxide/Internal/NativeMethods.cs
@@ -7096,6 +7096,18 @@ public static partial IntPtr pdf_oxide_element_get_text(
int index,
out int errorCode);
+ ///
+ /// Gets the §9.10.2 mapping-provenance label of an element
+ /// ("to_unicode"/"encoding"/"predefined_cmap"/"embedded_cmap"/
+ /// "actual_text"/"fallback"), or null when unknown.
+ ///
+ [LibraryImport(LibName, EntryPoint = "pdf_oxide_element_get_provenance", StringMarshalling = StringMarshalling.Utf8)]
+ [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvCdecl) })]
+ public static partial IntPtr pdf_oxide_element_get_provenance(
+ NativeHandle elements,
+ int index,
+ out int errorCode);
+
///
/// Gets the bounding rectangle of an element.
///
diff --git a/dart/lib/pdf_oxide.dart b/dart/lib/pdf_oxide.dart
index ca7a2cfa1..da44118cd 100644
--- a/dart/lib/pdf_oxide.dart
+++ b/dart/lib/pdf_oxide.dart
@@ -1169,6 +1169,8 @@ class _Native {
.lookupFunction<_ListStrC, _ListStrD>('pdf_oxide_element_get_type'),
elementGetText = lib
.lookupFunction<_ListStrC, _ListStrD>('pdf_oxide_element_get_text'),
+ elementGetProvenance = lib
+ .lookupFunction<_ListStrC, _ListStrD>('pdf_oxide_element_get_provenance'),
elementGetRect = lib.lookupFunction<_ElemRectC, _ElemRectD>(
'pdf_oxide_element_get_rect'),
elementsFree = lib
@@ -1680,6 +1682,7 @@ class _Native {
final _ElemCountD elementCount;
final _ListStrD elementGetType;
final _ListStrD elementGetText;
+ final _ListStrD elementGetProvenance;
final _ElemRectD elementGetRect;
final _ListFreeD elementsFree;
final _ElemJsonD elementsToJson;
@@ -7311,7 +7314,7 @@ class OcrEngine implements Finalizable {
/// A single layout element read from an [ElementList].
class Element {
- const Element(this.type, this.text, this.rect);
+ const Element(this.type, this.text, this.rect, [this.provenance = '']);
/// The element type label (e.g. `Text`, `Image`, `Table`).
final String type;
@@ -7322,6 +7325,12 @@ class Element {
/// The element's bounding box in page user-space points.
final Bbox rect;
+ /// ISO 32000-1 §9.10.2 mapping-provenance label (`to_unicode`/`encoding`/
+ /// `predefined_cmap`/`embedded_cmap`/`actual_text`/`fallback`), or `''` when
+ /// unknown. `fallback` means the text is a fabricated glyph-index echo, not
+ /// read from the file.
+ final String provenance;
+
@override
String toString() => 'Element($type, $rect)';
}
@@ -7372,6 +7381,8 @@ class ElementList implements Finalizable {
_check();
final type = _str(_n.elementGetType, index, 'elementGetType');
final text = _str(_n.elementGetText, index, 'elementGetText');
+ final provenance =
+ _str(_n.elementGetProvenance, index, 'elementGetProvenance');
final x = calloc();
final y = calloc();
final w = calloc();
@@ -7380,7 +7391,8 @@ class ElementList implements Finalizable {
try {
_n.elementGetRect(_handle, index, x, y, w, h, code);
if (code.value != 0) throw PdfOxideError(code.value, 'elementGetRect');
- return Element(type, text, Bbox(x.value, y.value, w.value, h.value));
+ return Element(
+ type, text, Bbox(x.value, y.value, w.value, h.value), provenance);
} finally {
calloc.free(x);
calloc.free(y);
diff --git a/go/pdf_oxide.go b/go/pdf_oxide.go
index 769c0fb56..612d3efb8 100644
--- a/go/pdf_oxide.go
+++ b/go/pdf_oxide.go
@@ -133,6 +133,7 @@ extern void* pdf_page_get_elements(void* handle, int32_t page_index, int* error_
extern int32_t pdf_oxide_element_count(const void* elements);
extern char* pdf_oxide_element_get_type(const void* elements, int32_t index, int* error_code);
extern char* pdf_oxide_element_get_text(const void* elements, int32_t index, int* error_code);
+extern char* pdf_oxide_element_get_provenance(const void* elements, int32_t index, int* error_code);
extern void pdf_oxide_element_get_rect(const void* elements, int32_t index, float* x, float* y, float* width, float* height, int* error_code);
extern void pdf_oxide_elements_free(void* handle);
diff --git a/go/types.go b/go/types.go
index 3ab8fdd43..323d935e4 100644
--- a/go/types.go
+++ b/go/types.go
@@ -282,6 +282,11 @@ type Element struct {
Y float32 `json:"y"`
Width float32 `json:"width"`
Height float32 `json:"height"`
+ // Provenance is the ISO 32000-1 §9.10.2 mapping tier the span's font
+ // offered ("to_unicode"/"encoding"/"predefined_cmap"/"embedded_cmap"/
+ // "actual_text"/"fallback"), or "" when unknown. "fallback" means the
+ // text is a fabricated glyph-index echo, not read from the file.
+ Provenance string `json:"provenance,omitempty"`
}
// ─── DocumentBuilder write-side value types ──────────────────────────────
diff --git a/include/pdf_oxide_c/pdf_oxide.h b/include/pdf_oxide_c/pdf_oxide.h
index 9e67ae80f..6c5e5751d 100644
--- a/include/pdf_oxide_c/pdf_oxide.h
+++ b/include/pdf_oxide_c/pdf_oxide.h
@@ -1465,6 +1465,19 @@ char *pdf_oxide_element_get_text(const FfiElementList *elements,
int32_t *error_code);
#endif
+#if !defined(PDF_OXIDE_TARGET_WASM32)
+/**
+ * The span's Unicode-mapping provenance label (ISO 32000-1 §9.10.2 tier):
+ * `"to_unicode"`, `"encoding"`, `"predefined_cmap"`, `"embedded_cmap"`,
+ * `"actual_text"`, or `"fallback"` (the text is a fabricated glyph-index echo,
+ * not read from the file). Returns null when the font could not be resolved.
+ * Free the returned string with `pdf_oxide_free_string`.
+ */
+char *pdf_oxide_element_get_provenance(const FfiElementList *elements,
+ int32_t index,
+ int32_t *error_code);
+#endif
+
#if !defined(PDF_OXIDE_TARGET_WASM32)
void pdf_oxide_element_get_rect(const FfiElementList *elements,
int32_t index,
diff --git a/objc/include/POXPdfOxide.h b/objc/include/POXPdfOxide.h
index 561e30e19..7530f9858 100644
--- a/objc/include/POXPdfOxide.h
+++ b/objc/include/POXPdfOxide.h
@@ -691,6 +691,11 @@ typedef struct {
- (nullable NSString*)typeAtIndex:(int32_t)index error:(NSError**)error;
/// The element text at `index` (nil on error).
- (nullable NSString*)textAtIndex:(int32_t)index error:(NSError**)error;
+/// The element ISO 32000-1 §9.10.2 mapping-provenance label at `index`
+/// (@"to_unicode"/@"encoding"/@"predefined_cmap"/@"embedded_cmap"/
+/// @"actual_text"/@"fallback"), or nil when unknown. @"fallback" means the text
+/// is a fabricated glyph-index echo, not read from the file.
+- (nullable NSString*)provenanceAtIndex:(int32_t)index error:(NSError**)error;
/// The element bounding box at `index`.
- (POXBbox)rectAtIndex:(int32_t)index error:(NSError**)error;
/// Serialize the whole list to JSON (nil on error).
diff --git a/objc/src/POXPdfOxide.m b/objc/src/POXPdfOxide.m
index 084b9ce13..2b751aa29 100644
--- a/objc/src/POXPdfOxide.m
+++ b/objc/src/POXPdfOxide.m
@@ -4498,6 +4498,12 @@ - (NSString*)textAtIndex:(int32_t)index error:(NSError**)error {
@"elementText", error);
}
+- (NSString*)provenanceAtIndex:(int32_t)index error:(NSError**)error {
+ int32_t code = 0;
+ return POXTakeString(pdf_oxide_element_get_provenance(_handle, index, &code), code,
+ @"elementProvenance", error);
+}
+
- (POXBbox)rectAtIndex:(int32_t)index error:(NSError**)error {
int32_t code = 0;
float x = 0, y = 0, w = 0, h = 0;
diff --git a/php/include/pdf_oxide.h b/php/include/pdf_oxide.h
index 05fba52f8..119b112fd 100644
--- a/php/include/pdf_oxide.h
+++ b/php/include/pdf_oxide.h
@@ -502,6 +502,10 @@ char *pdf_oxide_element_get_text(const FfiElementList *elements,
int32_t index,
int32_t *error_code);
+char *pdf_oxide_element_get_provenance(const FfiElementList *elements,
+ int32_t index,
+ int32_t *error_code);
+
void pdf_oxide_element_get_rect(const FfiElementList *elements,
int32_t index,
float *x,
diff --git a/src/bin/corpus_sig.rs b/src/bin/corpus_sig.rs
new file mode 100644
index 000000000..a34883a02
--- /dev/null
+++ b/src/bin/corpus_sig.rs
@@ -0,0 +1,84 @@
+//! Native Rust regression-sweep signature tool.
+//!
+//! Prints one line per PDF in a corpus directory:
+//!
+//! ```text
+//! \t\t\t\t
+//! ```
+//!
+//! Build once per version and diff the outputs to find extraction regressions:
+//!
+//! ```text
+//! cargo build --release --bin corpus_sig --jobs 3
+//! ./target/release/corpus_sig > head.txt
+//! # (in a v0.3.71 worktree) ./target/release/corpus_sig > base.txt
+//! diff base.txt head.txt
+//! ```
+//!
+//! Single process, release speed — no Python, no per-doc subprocess.
+
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+use std::path::{Path, PathBuf};
+
+use pdf_oxide::document::PdfDocument;
+
+fn collect_pdfs(dir: &Path, out: &mut Vec) {
+ if let Ok(entries) = std::fs::read_dir(dir) {
+ let mut entries: Vec<_> = entries.flatten().map(|e| e.path()).collect();
+ entries.sort();
+ for p in entries {
+ if p.is_dir() {
+ collect_pdfs(&p, out);
+ } else if p.extension().and_then(|e| e.to_str()) == Some("pdf") {
+ out.push(p);
+ }
+ }
+ }
+}
+
+fn main() {
+ let root = std::env::args()
+ .nth(1)
+ .expect("usage: corpus_sig ");
+ let root = PathBuf::from(root);
+ let mut pdfs = Vec::new();
+ collect_pdfs(&root, &mut pdfs);
+ eprintln!("corpus: {} pdfs", pdfs.len());
+
+ for pdf in &pdfs {
+ let rel = pdf.strip_prefix(&root).unwrap_or(pdf).display();
+ let doc = match PdfDocument::open(pdf) {
+ Ok(d) => d,
+ Err(_) => {
+ println!("{rel}\tOPEN_ERR\t0\t0\t[]");
+ continue;
+ },
+ };
+ let n = doc.page_count().unwrap_or(0);
+ let mut all_text = String::new();
+ let mut rots: Vec = Vec::with_capacity(n);
+ let mut max_word = 0usize;
+ for i in 0..n {
+ rots.push(doc.get_page_rotation(i).unwrap_or(0));
+ if let Ok(t) = doc.extract_text(i) {
+ all_text.push_str(&t);
+ all_text.push('\n');
+ }
+ if let Ok(words) = doc.extract_words(i) {
+ for w in &words {
+ max_word = max_word.max(w.text.chars().count());
+ }
+ }
+ }
+ let mut h = DefaultHasher::new();
+ all_text.hash(&mut h);
+ println!(
+ "{rel}\t{:016x}\t{}\t{}\t{:?}",
+ h.finish(),
+ all_text.chars().count(),
+ max_word,
+ rots
+ );
+ }
+}
diff --git a/src/converters/html.rs b/src/converters/html.rs
index 2c62af8ef..9211c26b6 100644
--- a/src/converters/html.rs
+++ b/src/converters/html.rs
@@ -780,6 +780,7 @@ mod tests {
// TextSpan represents complete strings from Tj/TJ operators, not individual chars
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "First".to_string(),
@@ -807,6 +808,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Second".to_string(),
diff --git a/src/converters/markdown.rs b/src/converters/markdown.rs
index cb2a54fcd..527c7bc7d 100644
--- a/src/converters/markdown.rs
+++ b/src/converters/markdown.rs
@@ -901,6 +901,7 @@ impl MarkdownConverter {
.iter()
.enumerate()
.map(|(seq, block)| TextSpan {
+ provenance: None,
text: block.text.clone(),
bbox: block.bbox,
font_name: block.dominant_font.clone(),
@@ -1554,6 +1555,7 @@ mod tests {
// Create spans with whitespace that should be filtered
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -1581,6 +1583,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " ".to_string(), // Whitespace only - should be filtered
@@ -1608,6 +1611,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
@@ -1657,6 +1661,7 @@ mod tests {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Section".to_string(),
@@ -1684,6 +1689,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "---".to_string(), // Punctuation only, but marked bold
@@ -1711,6 +1717,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Content".to_string(),
@@ -1764,6 +1771,7 @@ mod tests {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Year:".to_string(),
@@ -1791,6 +1799,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "2024".to_string(), // Numeric, should be bold if marked
@@ -1843,6 +1852,7 @@ mod tests {
// All potentially bolded
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Title".to_string(),
@@ -1870,6 +1880,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " ".to_string(), // Whitespace - should be filtered
@@ -1897,6 +1908,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "...".to_string(), // Punctuation - should be neutralized
@@ -1924,6 +1936,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " \n ".to_string(), // Mixed whitespace - should be filtered
@@ -1951,6 +1964,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Content".to_string(),
@@ -2139,6 +2153,7 @@ mod tests {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Content".to_string(),
@@ -2166,6 +2181,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " \n ".to_string(), // Whitespace with newlines
@@ -2193,6 +2209,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "More".to_string(),
diff --git a/src/document.rs b/src/document.rs
index dbb65a180..1ce74ba5c 100644
--- a/src/document.rs
+++ b/src/document.rs
@@ -8759,6 +8759,7 @@ impl PdfDocument {
};
spans.push(TextSpan {
+ provenance: None,
artifact_type: None,
text,
bbox: rect,
@@ -8929,6 +8930,7 @@ impl PdfDocument {
};
spans.push(TextSpan {
+ provenance: None,
artifact_type: None,
text,
bbox: rect,
@@ -17409,6 +17411,7 @@ impl PdfDocument {
let spans: Vec<_> = words
.into_iter()
.map(|w| crate::layout::TextSpan {
+ provenance: None,
artifact_type: None,
text: w.text,
bbox: w.bbox,
@@ -19214,6 +19217,7 @@ impl PdfDocument {
let word_spans: Vec = words
.into_iter()
.map(|w| crate::layout::TextSpan {
+ provenance: None,
artifact_type: None,
text: w.text,
bbox: w.bbox,
@@ -24379,6 +24383,7 @@ mod tests {
fn make_test_span(text: &str, x: f32, y: f32, width: f32, font_size: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -24910,6 +24915,7 @@ mod tests {
font_size: f32,
) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
text: text.to_string(),
bbox: crate::geometry::Rect {
@@ -28914,6 +28920,7 @@ mod tests {
fn make_span(label: &str, x: f32, y: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: label.to_string(),
@@ -28982,6 +28989,7 @@ mod tests {
use crate::geometry::Rect;
use crate::layout::{Color, FontWeight, TextSpan};
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -29675,6 +29683,7 @@ mod tests {
fn mk(text: &str, x: f32, y: f32, w: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -29749,6 +29758,7 @@ mod tests {
fn mk(text: &str, x: f32, y: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -29822,6 +29832,7 @@ mod tests {
fn mk(text: &str, x: f32, y: f32, w: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/elements/text.rs b/src/elements/text.rs
index 63ce076f7..d8783a3bb 100644
--- a/src/elements/text.rs
+++ b/src/elements/text.rs
@@ -147,6 +147,7 @@ impl From for TextContent {
impl From for TextSpan {
fn from(content: TextContent) -> Self {
TextSpan {
+ provenance: None,
text: content.text,
bbox: content.bbox,
font_name: content.font.name,
@@ -318,6 +319,7 @@ mod tests {
#[test]
fn test_text_span_conversion() {
let span = TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Test".to_string(),
diff --git a/src/extractors/gap_statistics.rs b/src/extractors/gap_statistics.rs
index 6f1c1361e..6032964ab 100644
--- a/src/extractors/gap_statistics.rs
+++ b/src/extractors/gap_statistics.rs
@@ -822,6 +822,7 @@ mod tests {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -849,6 +850,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
@@ -961,6 +963,7 @@ mod tests {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "A".to_string(),
@@ -988,6 +991,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "B".to_string(),
diff --git a/src/extractors/geometric_spacing.rs b/src/extractors/geometric_spacing.rs
index 724c0423f..8b08b40c7 100644
--- a/src/extractors/geometric_spacing.rs
+++ b/src/extractors/geometric_spacing.rs
@@ -146,6 +146,7 @@ mod tests {
fn make_span(text: &str, x: f32, width: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/extractors/text.rs b/src/extractors/text.rs
index 42479c414..02e99e5f8 100644
--- a/src/extractors/text.rs
+++ b/src/extractors/text.rs
@@ -3766,6 +3766,17 @@ impl<'doc> TextExtractor<'doc> {
}
}
+ // Attach the §9.10.2 mapping provenance now that each span's font name
+ // is finalized: the tier the span's font offered, or `Fallback` when it
+ // carries no mapping resource (the text is then a fabricated glyph-index
+ // echo, not read from the file). `None` when the font is unresolvable.
+ for span in self.spans.iter_mut() {
+ span.provenance = self
+ .fonts
+ .get(span.font_name.as_str())
+ .map(|f| f.best_mapping_provenance());
+ }
+
Ok(std::mem::take(&mut self.spans))
}
@@ -7376,6 +7387,7 @@ impl<'doc> TextExtractor<'doc> {
}
let span = TextSpan {
+ provenance: None,
text,
bbox: Rect {
x: buffer.user_pos_x,
@@ -7912,6 +7924,7 @@ impl<'doc> TextExtractor<'doc> {
// Step 5: Create TextSpan with primary_detected flag
let span = TextSpan {
+ provenance: None,
text: unicode_text,
bbox,
font_name: state
@@ -8631,6 +8644,7 @@ impl<'doc> TextExtractor<'doc> {
(effective_font_size, space_advance.abs())
};
let span = TextSpan {
+ provenance: None,
text: " ".to_string(),
bbox: Rect {
x: user_pos.x,
@@ -8808,6 +8822,7 @@ impl<'doc> TextExtractor<'doc> {
}
let span = TextSpan {
+ provenance: None,
text,
bbox: Rect {
x: buffer.user_pos_x,
@@ -9790,6 +9805,7 @@ mod tests {
fn test_split_boundary_merges_with_space() {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "the".to_string(),
@@ -9822,6 +9838,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "General".to_string(),
@@ -10710,6 +10727,7 @@ mod tests {
// being flaky.
fn snap_span(text: &str, x: f32, y: f32, w: f32, fs: f32, seq: usize) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -11691,6 +11709,7 @@ mod tests {
let mut extractor = TextExtractor::new();
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -11718,6 +11737,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -11770,6 +11790,7 @@ mod tests {
// body-text sizes.
let narrow_span =
|glyph: char, x: f32, font_size: f32, advance: f32, seq: usize| TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: glyph.to_string(),
@@ -11830,6 +11851,7 @@ mod tests {
let mut extractor = TextExtractor::new();
let narrow_at = |x: f32, seq: usize| TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "l".to_string(),
@@ -11886,6 +11908,7 @@ mod tests {
// Create spans all in one column
for i in 0..10 {
extractor.spans.push(TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: format!("Line {}", i),
@@ -12084,6 +12107,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -12111,6 +12135,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
@@ -12152,6 +12177,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -12179,6 +12205,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
@@ -12225,6 +12252,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Left".to_string(),
@@ -12252,6 +12280,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Right".to_string(),
@@ -12291,6 +12320,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -12318,6 +12348,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " ".to_string(),
@@ -12345,6 +12376,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
@@ -14778,6 +14810,7 @@ mod tests {
let mut extractor = TextExtractor::new();
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello World".to_string(), // >= 5 chars
@@ -14805,6 +14838,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello World".to_string(), // Same text, overlapping position
@@ -14842,6 +14876,7 @@ mod tests {
let mut extractor = TextExtractor::new();
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello World".to_string(),
@@ -14869,6 +14904,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello World".to_string(), // Same text but far apart
@@ -14966,6 +15002,7 @@ mod tests {
fn test_split_fused_words_camelcase() {
let mut extractor = TextExtractor::new();
extractor.spans = vec![TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "theGeneral".to_string(),
@@ -15004,6 +15041,7 @@ mod tests {
fn test_split_fused_words_no_split() {
let mut extractor = TextExtractor::new();
extractor.spans = vec![TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "hello".to_string(),
@@ -15189,6 +15227,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Right Col".to_string(),
@@ -15216,6 +15255,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Left Col".to_string(),
@@ -15299,6 +15339,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello ".to_string(), // ends with space
@@ -15326,6 +15367,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " World".to_string(), // starts with space
@@ -15629,6 +15671,7 @@ mod tests {
let mut extractor = TextExtractor::new();
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Line2".to_string(),
@@ -15656,6 +15699,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Line1".to_string(),
@@ -15865,6 +15909,7 @@ mod tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -15892,6 +15937,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: " ".to_string(), // offset_semantic space
@@ -16436,6 +16482,7 @@ mod profile_based_space_tests {
// Second value starts at x=131, creating a 1pt gap (100 + 30 = 130, gap = 1pt)
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "$0.00".to_string(),
@@ -16463,6 +16510,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "$0.00".to_string(),
@@ -16509,6 +16557,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "100".to_string(),
@@ -16536,6 +16585,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "200".to_string(),
@@ -16582,6 +16632,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hel".to_string(),
@@ -16609,6 +16660,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "lo".to_string(),
@@ -16661,6 +16713,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "123456".to_string(),
@@ -16688,6 +16741,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "72".to_string(),
@@ -16732,6 +16786,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "50".to_string(),
@@ -16759,6 +16814,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "00".to_string(),
@@ -16800,6 +16856,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -16827,6 +16884,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "72".to_string(),
@@ -16872,6 +16930,7 @@ mod profile_based_space_tests {
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "123456".to_string(),
@@ -16899,6 +16958,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "723".to_string(),
@@ -16939,6 +16999,7 @@ mod profile_based_space_tests {
/// Compact TextSpan builder for the intervening-ink decimal tests.
fn digit_test_span(text: &str, bbox: Rect, font_size: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -17070,6 +17131,7 @@ mod profile_based_space_tests {
// gap = 114.5 - (100.0 + 3.5) = 11.0pt -> 11.0 / 7.0 = 1.57x font size.
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "1".to_string(),
@@ -17097,6 +17159,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "0".to_string(),
@@ -17151,6 +17214,7 @@ mod profile_based_space_tests {
// gap = 238.4 - (200.0 + 24.0) = 14.4pt -> 14.4 / 12.0 = 1.2x font size.
extractor.spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "1234".to_string(),
@@ -17178,6 +17242,7 @@ mod profile_based_space_tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "56".to_string(),
diff --git a/src/ffi.rs b/src/ffi.rs
index 3a4b26e4a..f8060e77a 100644
--- a/src/ffi.rs
+++ b/src/ffi.rs
@@ -3146,6 +3146,33 @@ pub extern "C" fn pdf_oxide_element_get_text(
to_c_string(&list.spans[index as usize].text)
}
+/// The span's Unicode-mapping provenance label (ISO 32000-1 §9.10.2 tier):
+/// `"to_unicode"`, `"encoding"`, `"predefined_cmap"`, `"embedded_cmap"`,
+/// `"actual_text"`, or `"fallback"` (the text is a fabricated glyph-index echo,
+/// not read from the file). Returns null when the font could not be resolved.
+/// Free the returned string with `pdf_oxide_free_string`.
+#[no_mangle]
+pub extern "C" fn pdf_oxide_element_get_provenance(
+ elements: *const FfiElementList,
+ index: i32,
+ error_code: *mut i32,
+) -> *mut c_char {
+ if elements.is_null() || index < 0 {
+ set_error(error_code, ERR_INVALID_ARG);
+ return ptr::null_mut();
+ }
+ let list = handle_ref(elements);
+ if (index as usize) >= list.spans.len() {
+ set_error(error_code, ERR_INVALID_PAGE);
+ return ptr::null_mut();
+ }
+ set_error(error_code, ERR_SUCCESS);
+ match list.spans[index as usize].provenance {
+ Some(p) => to_c_string(p.as_str()),
+ None => ptr::null_mut(),
+ }
+}
+
#[no_mangle]
pub extern "C" fn pdf_oxide_element_get_rect(
elements: *const FfiElementList,
@@ -8765,6 +8792,9 @@ struct JsonElement<'a> {
y: f32,
width: f32,
height: f32,
+ /// §9.10.2 mapping-provenance label; omitted when the font is unresolved.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ provenance: Option<&'static str>,
}
#[derive(Serialize)]
@@ -8947,6 +8977,7 @@ pub extern "C" fn pdf_oxide_elements_to_json(
y: s.bbox.y,
width: s.bbox.width,
height: s.bbox.height,
+ provenance: s.provenance.map(|p| p.as_str()),
})
.collect();
match serde_json::to_string(&items) {
diff --git a/src/fonts/font_dict.rs b/src/fonts/font_dict.rs
index 4a7e98c01..9f103796f 100644
--- a/src/fonts/font_dict.rs
+++ b/src/fonts/font_dict.rs
@@ -377,6 +377,56 @@ impl FontInfo {
self.truetype_cmap().is_some()
}
+ /// The most authoritative Unicode-mapping resource this font offers, as a
+ /// [`MappingProvenance`](crate::fonts::MappingProvenance).
+ ///
+ /// This is a **fact** derived from the font's structure — which mapping
+ /// resources exist — not a decode of any particular character code. It
+ /// mirrors the ISO 32000-1 §9.10.2 priority order and covers every font
+ /// type, so it is complete where a font-type-specific structural check is
+ /// not.
+ ///
+ /// [`Fallback`](crate::fonts::MappingProvenance::Fallback) is the important
+ /// value: it means the font carries **no** mapping resource — no usable
+ /// `/ToUnicode`, no predefined CID→Unicode collection, no embedded `cmap`,
+ /// and no simple-font encoding — so any Unicode extracted for its glyphs is
+ /// a fabricated echo, not read from the file (§9.10.2: "there is no way to
+ /// determine what the character code represents"). Callers compose their own
+ /// policy from this (route to OCR, flag the page, keep the raw echo).
+ pub fn best_mapping_provenance(&self) -> crate::fonts::MappingProvenance {
+ use crate::fonts::MappingProvenance as P;
+ // 1. A present, non-empty /ToUnicode CMap is authoritative (§9.10.2).
+ if self
+ .to_unicode
+ .as_ref()
+ .and_then(|c| c.get())
+ .is_some_and(|m| !m.is_empty())
+ {
+ return P::ToUnicode;
+ }
+ // 2. A predefined CID→Unicode collection: a Type0 font whose descendant
+ // uses a known, non-Identity ordering (Adobe-GB1/CNS1/Japan1/Korea1).
+ if self.subtype == "Type0" {
+ if let Some(info) = &self.cid_system_info {
+ if info.ordering != "Identity" && !info.ordering.is_empty() {
+ return P::PredefinedCMap;
+ }
+ }
+ }
+ // 3. The embedded program's own cmap (recoverable byte-as-GID / Identity
+ // subsets that kept a usable cmap).
+ if self.has_truetype_cmap() {
+ return P::EmbeddedCmap;
+ }
+ // 4. A simple font resolves through its /Encoding → glyph name → AGL, and
+ // symbolic Symbol/ZapfDingbats through their built-in encodings.
+ if self.subtype != "Type0" {
+ return P::EncodingName;
+ }
+ // 5. A Type0 font with none of the above severs every path to Unicode.
+ P::Fallback
+ }
+
/// Look up the embedded font program's `post`-table glyph name for the
/// given GID.
///
@@ -8169,6 +8219,59 @@ mod tests {
f
}
+ // =========================================================================
+ // best_mapping_provenance — font-level Unicode-mapping capability (fact)
+ // =========================================================================
+
+ // The critical case: a Type0 / Identity-ordered subset with no ToUnicode
+ // and no embedded cmap severs every path to Unicode → Fallback.
+ #[test]
+ fn best_mapping_provenance_fallback_on_severed_identity_type0() {
+ let f = make_font(|f| {
+ f.subtype = "Type0".to_string();
+ f.encoding = Encoding::Standard("Identity-H".to_string());
+ f.cid_system_info = Some(CIDSystemInfo {
+ registry: "Adobe".to_string(),
+ ordering: "Identity".to_string(),
+ supplement: 0,
+ });
+ f.to_unicode = None;
+ });
+ assert_eq!(f.best_mapping_provenance(), crate::fonts::MappingProvenance::Fallback);
+ }
+
+ #[test]
+ fn best_mapping_provenance_fallback_type0_without_cidsysteminfo() {
+ let f = make_font(|f| {
+ f.subtype = "Type0".to_string();
+ f.encoding = Encoding::Standard("Identity-H".to_string());
+ f.cid_system_info = None;
+ f.to_unicode = None;
+ });
+ assert_eq!(f.best_mapping_provenance(), crate::fonts::MappingProvenance::Fallback);
+ }
+
+ // A known character collection (non-Identity ordering) → predefined CMap.
+ #[test]
+ fn best_mapping_provenance_predefined_for_known_collection() {
+ let f = make_font(|f| {
+ f.subtype = "Type0".to_string();
+ f.cid_system_info = Some(CIDSystemInfo {
+ registry: "Adobe".to_string(),
+ ordering: "Japan1".to_string(),
+ supplement: 6,
+ });
+ });
+ assert_eq!(f.best_mapping_provenance(), crate::fonts::MappingProvenance::PredefinedCMap);
+ }
+
+ // A simple font resolves through its /Encoding → glyph name → AGL.
+ #[test]
+ fn best_mapping_provenance_encoding_for_simple_font() {
+ let f = make_font(|_| {});
+ assert_eq!(f.best_mapping_provenance(), crate::fonts::MappingProvenance::EncodingName);
+ }
+
// =========================================================================
// get_space_glyph_width — the space advance drives the geometric word-gap
// threshold, so it must be a REAL space advance, never an arbitrary glyph.
diff --git a/src/fonts/mod.rs b/src/fonts/mod.rs
index c1a972431..2451aa2c3 100644
--- a/src/fonts/mod.rs
+++ b/src/fonts/mod.rs
@@ -32,6 +32,7 @@ pub mod non_text_detection;
/// faces (Ryumin-Light, GothicBBB-Medium, STSong-Light, …) without embedding
/// glyph outlines. ISO 32000-2 §9.7.5.2 mandates support for these collections.
pub mod predefined_cidfont;
+pub mod provenance;
/// TrueType font CMap parsing for glyph-to-character mapping.
pub mod truetype_cmap;
/// TrueType/OpenType font parser for PDF embedding (v0.3.0).
@@ -50,5 +51,6 @@ pub use font_subsetter::{subset_font_bytes, FontSubsetter, GlyphRemapper, Subset
pub use non_text_detection::{
CharacterConfidence, ConfidenceReason, NonTextDetector, NonTextStats,
};
+pub use provenance::MappingProvenance;
pub use truetype_cmap::TrueTypeCMap;
pub use truetype_parser::{FontMetrics, TrueTypeError, TrueTypeFont, TrueTypeResult};
diff --git a/src/fonts/non_text_detection.rs b/src/fonts/non_text_detection.rs
index 6299b4ca1..bede98387 100644
--- a/src/fonts/non_text_detection.rs
+++ b/src/fonts/non_text_detection.rs
@@ -415,6 +415,7 @@ mod tests {
use crate::geometry::Rect;
use crate::layout::{Color, FontWeight, TextSpan};
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/fonts/provenance.rs b/src/fonts/provenance.rs
new file mode 100644
index 000000000..bdd9ac280
--- /dev/null
+++ b/src/fonts/provenance.rs
@@ -0,0 +1,153 @@
+//! Provenance of a decoded character's Unicode value.
+//!
+//! [`MappingProvenance`] records *which tier of the ISO 32000-1 §9.10.2
+//! character-to-Unicode cascade produced a character's Unicode value* — or that
+//! none did and the value was chosen by fallback. It is a **fact** about how the
+//! value was derived, not a **judgment** about whether the text is "correct" or
+//! "corrupt".
+//!
+//! The library deliberately exposes this fact rather than shipping the
+//! judgments callers build from it. From provenance a caller composes their own
+//! policy — route a page to OCR, flag low-confidence runs, detect a "text layer
+//! that can't be read", or keep the raw glyph-index echo for an intentional
+//! payload — without the library having to add a flag per use case. See §9.10.2:
+//! when every method fails, "there is no way to determine what the character
+//! code represents … a conforming reader may choose a character code of their
+//! choosing" — that case is [`MappingProvenance::Fallback`].
+//!
+//! Variants are ordered most-authoritative to least. [`MappingProvenance::rank`]
+//! exposes that order so a multi-character span can be summarised by its
+//! *weakest* character (the honest signal for "does this run contain anything
+//! fabricated?").
+
+/// Which §9.10.2 tier produced a character's Unicode value.
+///
+/// Serializes as the same stable lowercase labels [`Self::as_str`] returns, so
+/// every serde-based binding (WASM/JSON) and every explicit accessor agree.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
+#[non_exhaustive]
+pub enum MappingProvenance {
+ /// An `/ActualText` replacement on a structure element or marked-content
+ /// sequence (§14.9.4). An explicit, authoritative override of the shown
+ /// characters.
+ #[serde(rename = "actual_text")]
+ ActualText,
+ /// The font's `/ToUnicode` CMap (§9.10.3) — the authoritative per-font map.
+ #[serde(rename = "to_unicode")]
+ ToUnicode,
+ /// The font `/Encoding` → glyph name → Adobe Glyph List path (§9.10.2, the
+ /// simple-font branch).
+ #[serde(rename = "encoding")]
+ EncodingName,
+ /// A predefined CID→Unicode CMap for a known character collection (§9.10.2,
+ /// e.g. `Adobe-Japan1-UCS2`).
+ #[serde(rename = "predefined_cmap")]
+ PredefinedCMap,
+ /// Inversion of the embedded font program's own `cmap` table — the
+ /// recoverable byte-as-GID / Identity subset shape.
+ #[serde(rename = "embedded_cmap")]
+ EmbeddedCmap,
+ /// No mapping tier produced a value; the character was chosen by fallback
+ /// (a CID-as-Unicode echo, or `U+FFFD`). Per §9.10.2 the character code's
+ /// meaning is undetermined, so any Unicode here is **fabricated by the
+ /// extractor, not read from the file**.
+ #[serde(rename = "fallback")]
+ Fallback,
+}
+
+impl MappingProvenance {
+ /// Authority rank, `0` = most authoritative (`ActualText`) … `5` = least
+ /// (`Fallback`). Lower is stronger, so `max` over a span's characters yields
+ /// the span's weakest provenance.
+ #[must_use]
+ pub fn rank(self) -> u8 {
+ match self {
+ Self::ActualText => 0,
+ Self::ToUnicode => 1,
+ Self::EncodingName => 2,
+ Self::PredefinedCMap => 3,
+ Self::EmbeddedCmap => 4,
+ Self::Fallback => 5,
+ }
+ }
+
+ /// Was this value actually read from the file (any real mapping tier), as
+ /// opposed to fabricated by the [`Fallback`](Self::Fallback) path?
+ #[must_use]
+ pub fn is_from_file(self) -> bool {
+ !matches!(self, Self::Fallback)
+ }
+
+ /// A stable, lowercase label for bindings and serialized surfaces. Shared
+ /// so every language binding exposes the same strings:
+ /// `"actual_text"`, `"to_unicode"`, `"encoding"`, `"predefined_cmap"`,
+ /// `"embedded_cmap"`, `"fallback"`.
+ #[must_use]
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::ActualText => "actual_text",
+ Self::ToUnicode => "to_unicode",
+ Self::EncodingName => "encoding",
+ Self::PredefinedCMap => "predefined_cmap",
+ Self::EmbeddedCmap => "embedded_cmap",
+ Self::Fallback => "fallback",
+ }
+ }
+
+ /// The weaker (less authoritative) of two provenances — the reduction used
+ /// to summarise a run of characters by its least-trustworthy member.
+ #[must_use]
+ pub fn weaker(self, other: Self) -> Self {
+ if self.rank() >= other.rank() {
+ self
+ } else {
+ other
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rank_orders_most_to_least_authoritative() {
+ let ordered = [
+ MappingProvenance::ActualText,
+ MappingProvenance::ToUnicode,
+ MappingProvenance::EncodingName,
+ MappingProvenance::PredefinedCMap,
+ MappingProvenance::EmbeddedCmap,
+ MappingProvenance::Fallback,
+ ];
+ for pair in ordered.windows(2) {
+ assert!(pair[0].rank() < pair[1].rank(), "{:?} must outrank {:?}", pair[0], pair[1]);
+ }
+ }
+
+ #[test]
+ fn only_fallback_is_not_from_file() {
+ assert!(!MappingProvenance::Fallback.is_from_file());
+ for p in [
+ MappingProvenance::ActualText,
+ MappingProvenance::ToUnicode,
+ MappingProvenance::EncodingName,
+ MappingProvenance::PredefinedCMap,
+ MappingProvenance::EmbeddedCmap,
+ ] {
+ assert!(p.is_from_file(), "{p:?} is a real mapping tier");
+ }
+ }
+
+ #[test]
+ fn weaker_summarises_a_run_by_its_least_trustworthy_char() {
+ // A span with one fabricated char is, as a whole, fabricated.
+ let span = MappingProvenance::ToUnicode
+ .weaker(MappingProvenance::ToUnicode)
+ .weaker(MappingProvenance::Fallback);
+ assert_eq!(span, MappingProvenance::Fallback);
+ // A clean span keeps its strongest-consistent provenance.
+ let clean = MappingProvenance::ToUnicode.weaker(MappingProvenance::ActualText);
+ assert_eq!(clean, MappingProvenance::ToUnicode);
+ }
+}
diff --git a/src/layout/font_normalization.rs b/src/layout/font_normalization.rs
index bda2f26a6..b4f666d28 100644
--- a/src/layout/font_normalization.rs
+++ b/src/layout/font_normalization.rs
@@ -112,6 +112,7 @@ mod tests {
fn make_span(text: &str, bold: bool) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/layout/text_block.rs b/src/layout/text_block.rs
index 401b1eac9..ce103d325 100644
--- a/src/layout/text_block.rs
+++ b/src/layout/text_block.rs
@@ -148,6 +148,17 @@ pub struct TextSpan {
/// both use base-form characters (no presentation forms, no `/ReversedChars`).
#[serde(skip_serializing_if = "is_false", default)]
pub rtl_draw_logical: bool,
+ /// Provenance of this span's Unicode text — which ISO 32000-1 §9.10.2
+ /// mapping tier the font offered, as a
+ /// [`MappingProvenance`](crate::fonts::MappingProvenance). `None` when the
+ /// span was not produced by the extractor (synthetic/test spans). A
+ /// [`Fallback`](crate::fonts::MappingProvenance::Fallback) value means the
+ /// font carried no mapping resource, so the text is a fabricated glyph-index
+ /// echo, not read from the file. Runtime metadata only — deliberately not
+ /// serialized (kept out of span JSON so existing output is byte-identical);
+ /// bindings surface it through explicit accessors.
+ #[serde(skip_serializing_if = "Option::is_none", default)]
+ pub provenance: Option,
}
/// serde skip helper: omit a `false` flag (the common case) from serialized output.
@@ -195,6 +206,7 @@ impl Default for TextSpan {
wmode: 0,
text_rise: 0.0,
rtl_draw_logical: false,
+ provenance: None,
}
}
}
@@ -820,6 +832,28 @@ impl TextLine {
mod tests {
use super::*;
+ // The provenance fact reaches every JSON/serde binding (WASM, Go, Ruby,
+ // Java's structured extraction, ...) through span serialization: present as
+ // a stable label when known, omitted when absent so existing output is
+ // byte-identical.
+ #[test]
+ fn provenance_serializes_as_stable_label_and_omits_when_absent() {
+ let mut span = TextSpan {
+ text: "x".to_string(),
+ ..TextSpan::default()
+ };
+ span.provenance = Some(crate::fonts::MappingProvenance::Fallback);
+ let json = serde_json::to_string(&span).unwrap();
+ assert!(json.contains("\"provenance\":\"fallback\""), "got {json}");
+
+ let plain = TextSpan {
+ text: "y".to_string(),
+ ..TextSpan::default()
+ };
+ let json = serde_json::to_string(&plain).unwrap();
+ assert!(!json.contains("provenance"), "absent provenance must be omitted: {json}");
+ }
+
fn mock_char(c: char, x: f32, y: f32) -> TextChar {
let bbox = Rect::new(x, y, 10.0, 12.0);
TextChar {
diff --git a/src/ocr/engine.rs b/src/ocr/engine.rs
index 89c469d2a..d9430ffd4 100644
--- a/src/ocr/engine.rs
+++ b/src/ocr/engine.rs
@@ -57,6 +57,7 @@ impl OcrSpan {
let font_size = self.estimate_font_size(height_pixels, scale);
TextSpan {
+ provenance: None,
artifact_type: None,
text: self.text.clone(),
bbox,
diff --git a/src/pipeline/converters/html.rs b/src/pipeline/converters/html.rs
index a6d0578d4..7877739dd 100644
--- a/src/pipeline/converters/html.rs
+++ b/src/pipeline/converters/html.rs
@@ -758,6 +758,7 @@ mod tests {
) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -1151,6 +1152,7 @@ mod tests {
italic: bool,
) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/converters/markdown.rs b/src/pipeline/converters/markdown.rs
index feb1f7264..4003b4891 100644
--- a/src/pipeline/converters/markdown.rs
+++ b/src/pipeline/converters/markdown.rs
@@ -4147,6 +4147,7 @@ mod tests {
) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -4186,6 +4187,7 @@ mod tests {
) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -4938,6 +4940,7 @@ mod tests {
) -> OrderedTextSpan {
let mut s = OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -5711,6 +5714,7 @@ mod tests {
#[test]
fn test_issue8_table_cell_renders_bold_marker() {
let bold_span = TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Critical".to_string(),
diff --git a/src/pipeline/converters/plain_text.rs b/src/pipeline/converters/plain_text.rs
index 940dd7fb6..5bb88b5ec 100644
--- a/src/pipeline/converters/plain_text.rs
+++ b/src/pipeline/converters/plain_text.rs
@@ -667,6 +667,7 @@ mod tests {
fn make_span(text: &str, x: f32, y: f32) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -855,6 +856,7 @@ mod tests {
fn make_span_with_width(text: &str, x: f32, y: f32, width: f32) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/ordered_span.rs b/src/pipeline/ordered_span.rs
index 7ea5196b6..a64336058 100644
--- a/src/pipeline/ordered_span.rs
+++ b/src/pipeline/ordered_span.rs
@@ -379,6 +379,7 @@ mod tests {
fn make_span(text: &str, x: f32, y: f32, w: f32, h: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/reading_order/geometric.rs b/src/pipeline/reading_order/geometric.rs
index d3a17ca1d..ea10b32f3 100644
--- a/src/pipeline/reading_order/geometric.rs
+++ b/src/pipeline/reading_order/geometric.rs
@@ -330,6 +330,7 @@ mod tests {
fn make_span(text: &str, x: f32, y: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/reading_order/simple.rs b/src/pipeline/reading_order/simple.rs
index 46470c628..b37d00d96 100644
--- a/src/pipeline/reading_order/simple.rs
+++ b/src/pipeline/reading_order/simple.rs
@@ -53,6 +53,7 @@ mod tests {
fn make_span(text: &str, x: f32, y: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/reading_order/structure_tree.rs b/src/pipeline/reading_order/structure_tree.rs
index dabc2e9a8..910da06fe 100644
--- a/src/pipeline/reading_order/structure_tree.rs
+++ b/src/pipeline/reading_order/structure_tree.rs
@@ -225,6 +225,7 @@ mod tests {
fn make_span(text: &str, x: f32, y: f32, mcid: Option) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/pipeline/reading_order/xycut.rs b/src/pipeline/reading_order/xycut.rs
index 3d4c47a2f..d4ea1fa90 100644
--- a/src/pipeline/reading_order/xycut.rs
+++ b/src/pipeline/reading_order/xycut.rs
@@ -2140,6 +2140,7 @@ mod tests {
use crate::layout::{Color, FontWeight};
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/python.rs b/src/python.rs
index daece3773..f96b89b5e 100644
--- a/src/python.rs
+++ b/src/python.rs
@@ -4205,6 +4205,15 @@ impl PyTextSpan {
fn sequence(&self) -> usize {
self.inner.sequence
}
+ /// Which ISO 32000-1 §9.10.2 mapping tier the span's font offered:
+ /// `"to_unicode"`, `"encoding"`, `"predefined_cmap"`, `"embedded_cmap"`,
+ /// `"actual_text"`, or `"fallback"` — the last meaning the font carried no
+ /// mapping resource, so the text is a fabricated glyph-index echo rather
+ /// than read from the file. `None` when the font could not be resolved.
+ #[getter]
+ fn provenance(&self) -> Option<&'static str> {
+ self.inner.provenance.map(|p| p.as_str())
+ }
}
#[pyclass(module = "pdf_oxide.pdf_oxide", name = "TextWord", skip_from_py_object)]
diff --git a/src/search/text_search.rs b/src/search/text_search.rs
index ad76419c7..2e831a7d6 100644
--- a/src/search/text_search.rs
+++ b/src/search/text_search.rs
@@ -316,6 +316,7 @@ mod tests {
fn test_build_text_with_positions() {
let spans = vec![
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "Hello".to_string(),
@@ -347,6 +348,7 @@ mod tests {
rtl_draw_logical: false,
},
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: "World".to_string(),
diff --git a/src/structure/spatial_table_detector.rs b/src/structure/spatial_table_detector.rs
index 068ec7c46..0e25cb3e5 100644
--- a/src/structure/spatial_table_detector.rs
+++ b/src/structure/spatial_table_detector.rs
@@ -4600,6 +4600,7 @@ mod tests {
fn create_test_span(text: &str, x: f32, y: f32, width: f32, height: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -7107,6 +7108,7 @@ mod tests {
/// rule actually reads: bbox, font_size, and text.
fn ts(text: &str, x: f32, y: f32, width: f32, fs: f32) -> TextSpan {
TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
diff --git a/src/structure/table_extractor.rs b/src/structure/table_extractor.rs
index 339a3a441..4d9b2c067 100644
--- a/src/structure/table_extractor.rs
+++ b/src/structure/table_extractor.rs
@@ -794,6 +794,7 @@ fn extract_cell(
block.text.clone()
};
cell_spans.push(TextSpan {
+ provenance: None,
artifact_type: None,
text: span_text,
bbox: block.bbox,
@@ -1152,6 +1153,7 @@ mod tests {
use crate::layout::text_block::{Color, FontWeight};
crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: text.to_string(),
@@ -1347,6 +1349,7 @@ mod tests {
// "（" x=353.83 w=10.56 end=364.39 gap=-0.18 (overlap → no space)
// "peu/d" x=364.39 w=25.24 gap=0.00 (touching → no space)
let base = crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: String::new(),
@@ -1424,6 +1427,7 @@ mod tests {
table_elem.add_child(StructChild::StructElem(Box::new(tr)));
let base = crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: String::new(),
@@ -1500,6 +1504,7 @@ mod tests {
table_elem.add_child(StructChild::StructElem(Box::new(tr)));
let base = crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: String::new(),
@@ -1602,6 +1607,7 @@ mod tests {
table_elem.add_child(StructChild::StructElem(Box::new(tr)));
let base = crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: String::new(),
@@ -1687,6 +1693,7 @@ mod tests {
table_elem.add_child(StructChild::StructElem(Box::new(tr)));
let base = crate::layout::TextSpan {
+ provenance: None,
text_rise: 0.0,
artifact_type: None,
text: String::new(),
diff --git a/swift/Sources/PdfOxide/PdfOxide.swift b/swift/Sources/PdfOxide/PdfOxide.swift
index 006366d05..3ebf984a7 100644
--- a/swift/Sources/PdfOxide/PdfOxide.swift
+++ b/swift/Sources/PdfOxide/PdfOxide.swift
@@ -3406,6 +3406,11 @@ public struct Element {
public let type: String
public let text: String
public let rect: Bbox
+ /// ISO 32000-1 §9.10.2 mapping-provenance label ("to_unicode"/"encoding"/
+ /// "predefined_cmap"/"embedded_cmap"/"actual_text"/"fallback"), or nil when
+ /// unknown. "fallback" means the text is a fabricated glyph-index echo, not
+ /// read from the file.
+ public let provenance: String?
}
/// An opaque list of page elements (`FfiElementList`), freed in `deinit`/`close()`.
@@ -3435,9 +3440,14 @@ public final class ElementList {
pdf_oxide_element_get_text(h, idx, &code), code, "ElementList.text")
var x: Float = 0, y: Float = 0, w: Float = 0, hgt: Float = 0
pdf_oxide_element_get_rect(h, idx, &x, &y, &w, &hgt, &code)
+ var pcode: Int32 = 0
+ let provPtr = pdf_oxide_element_get_provenance(h, idx, &pcode)
+ let provenance: String? =
+ provPtr != nil ? try takeString(provPtr, pcode, "ElementList.provenance") : nil
return Element(
type: type, text: text,
- rect: Bbox(x: Double(x), y: Double(y), width: Double(w), height: Double(hgt))
+ rect: Bbox(x: Double(x), y: Double(y), width: Double(w), height: Double(hgt)),
+ provenance: provenance
)
}
diff --git a/tests/test_pipeline_html_converter.rs b/tests/test_pipeline_html_converter.rs
index 929f8d1c0..e1f300f80 100644
--- a/tests/test_pipeline_html_converter.rs
+++ b/tests/test_pipeline_html_converter.rs
@@ -23,6 +23,7 @@ use pdf_oxide::pipeline::{OrderedTextSpan, TextPipelineConfig};
fn make_span(text: &str, x: f32, y: f32, font_size: f32, weight: FontWeight) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
artifact_type: None,
text: text.to_string(),
bbox: Rect::new(x, y, 50.0, font_size),
@@ -64,6 +65,7 @@ fn make_span_with_color(
) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
artifact_type: None,
text: text.to_string(),
bbox: Rect::new(x, y, 50.0, font_size),
@@ -105,6 +107,7 @@ fn make_span_italic(
) -> OrderedTextSpan {
OrderedTextSpan::new(
TextSpan {
+ provenance: None,
artifact_type: None,
text: text.to_string(),
bbox: Rect::new(x, y, 50.0, font_size),
diff --git a/tests/test_spacing_integration.rs b/tests/test_spacing_integration.rs
index 0572ef4c4..50fbd2779 100644
--- a/tests/test_spacing_integration.rs
+++ b/tests/test_spacing_integration.rs
@@ -33,6 +33,7 @@ fn create_test_span(
_tj_space_signal: bool,
) -> TextSpan {
TextSpan {
+ provenance: None,
artifact_type: None,
text: text.to_string(),
bbox: Rect::new(x, y, width, height),
diff --git a/tests/test_spacing_spec_compliant.rs b/tests/test_spacing_spec_compliant.rs
index 816295e5c..7e3108613 100644
--- a/tests/test_spacing_spec_compliant.rs
+++ b/tests/test_spacing_spec_compliant.rs
@@ -22,6 +22,7 @@ use pdf_oxide::layout::{Color, FontWeight, TextSpan};
/// Create a test text span with specified position and dimensions.
fn create_test_span(text: &str, x: f32, y: f32, width: f32, height: f32) -> TextSpan {
TextSpan {
+ provenance: None,
artifact_type: None,
text: text.to_string(),
bbox: Rect::new(x, y, width, height),
diff --git a/zig/lib/pdf_oxide.zig b/zig/lib/pdf_oxide.zig
index d763dba2c..b1e4264a3 100644
--- a/zig/lib/pdf_oxide.zig
+++ b/zig/lib/pdf_oxide.zig
@@ -4073,6 +4073,17 @@ pub const ElementList = struct {
return takeString(alloc, c.pdf_oxide_element_get_text(h, index, &code), code);
}
+ /// Element ISO 32000-1 §9.10.2 mapping-provenance label at `index`
+ /// ("to_unicode"/"encoding"/"predefined_cmap"/"embedded_cmap"/"actual_text"/
+ /// "fallback"), or empty when unknown; caller owns the returned slice.
+ /// "fallback" means the text is a fabricated glyph-index echo, not read from
+ /// the file.
+ pub fn getProvenance(self: ElementList, alloc: std.mem.Allocator, index: i32) Error![]u8 {
+ const h = try self.live();
+ var code: i32 = 0;
+ return takeString(alloc, c.pdf_oxide_element_get_provenance(h, index, &code), code);
+ }
+
/// Element bounding box at `index`.
pub fn getRect(self: ElementList, index: i32) Error!Bbox {
const h = try self.live();