feat(extractor): geometric underline detection on TextItem (#116)

* feat(extractor): geometric underline detection on TextItem (ENG-5015)

PDFs carry no underline font flag — underlines are stroked horizontal
lines or thin filled rects drawn under the baseline. Correlate those
graphics (already parsed from the content stream) with text items in a
post-pass: a rule within ~0.35em below the baseline covering >=60% of
an item's width marks is_underline.

Exposed through the napi and python bindings. Verified on real docs:
4/4 underlined sentences flagged on a Japanese report, links/headings
flagged on 8 of 10 underline-bearing eval docs, zero flags on docs
without underlines. Known FP source (table cell borders) documented —
downstream applies inline styling only to plain-text regions.

napi 1.9.8 -> 1.9.9.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(extractor): underline rules only from painted rects, normalized extents (review)

Two review fixes: (1) normalize rect extents before the thickness/width
checks — `re` operands pass through the CTM so width/height can be
negative, which missed negative-width rules and let negative-height
bands pass as thin; (2) only feed painted rects to underline detection —
`re` rects now wait in a pending list until a paint operator (S/s, f/F/
f*, B/B*/b/b*) confirms them, and `re W n` clip-only paths are discarded
at `n`, so invisible clip boundaries no longer underline nearby text.
Marking moved into content_stream where paint state lives (pre-rotation,
consistent device space).

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(extractor): harden underline detection

* feat(cli): export positioned text item json

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: abimaelmartell

README.md

@@ -111,6 +111,9 @@ pdf2md document.pdf
 # JSON output (for piping)
 pdf2md document.pdf --json
 
+# Positioned TextItem JSON, including is_underline metadata
+pdf2md document.pdf --items-json
+
 # Raw markdown only (no headers)
 pdf2md document.pdf --raw
 

napi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firecrawl/pdf-inspector",
-  "version": "1.9.8",
+  "version": "1.9.9",
   "description": "Fast PDF classification and text extraction. Detect text-based vs scanned PDFs, extract text by region with quality checks. Native Rust performance via napi-rs.",
   "main": "index.js",
   "types": "index.d.ts",

napi/src/lib.rs

@@ -80,6 +80,9 @@ pub struct TextItem {
     pub page: u32,
     pub is_bold: bool,
     pub is_italic: bool,
+    /// Underline detected geometrically (drawn rule/thin rect under the
+    /// baseline) — PDFs carry no underline font flag.
+    pub is_underline: bool,
     pub item_type: ItemType,
     /// URL for link items, `None` for other types.
     pub link_url: Option<String>,
@@ -290,6 +293,7 @@ pub fn extract_text_with_positions(
                     page: item.page,
                     is_bold: item.is_bold,
                     is_italic: item.is_italic,
+                    is_underline: item.is_underline,
                     item_type,
                     link_url,
                 }

pdf_inspector.pyi

@@ -38,6 +38,7 @@ class TextItem:
     page: int
     is_bold: bool
     is_italic: bool
+    is_underline: bool
     item_type: str
 
 class RegionText:

src/bin/pdf2md.rs

@@ -1,6 +1,10 @@
 //! CLI tool for PDF to Markdown conversion
 
-use pdf_inspector::{process_pdf_with_options, LayoutComplexity, PdfOptions, PdfType, ProcessMode};
+use pdf_inspector::extractor::ItemType;
+use pdf_inspector::{
+    extract_text_with_positions_pages, process_pdf_with_options, LayoutComplexity, PdfOptions,
+    PdfType, ProcessMode, TextItem,
+};
 use std::collections::HashSet;
 use std::env;
 use std::fmt::Write;
@@ -47,6 +51,92 @@ fn format_ocr_reasons_by_page(reasons: &[pdf_inspector::PageOcrReasons]) -> Stri
         .join(",")
 }
 
+fn item_type_label(item_type: &ItemType) -> &'static str {
+    match item_type {
+        ItemType::Text => "text",
+        ItemType::Image => "image",
+        ItemType::Link(_) => "link",
+        ItemType::FormField => "form_field",
+    }
+}
+
+fn format_items_json(items: &[TextItem]) -> String {
+    let underlined_count = items.iter().filter(|item| item.is_underline).count();
+    let items_json = items
+        .iter()
+        .map(|item| {
+            let mcid = item
+                .mcid
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "null".to_string());
+            let link_url = match &item.item_type {
+                ItemType::Link(url) => format!(r#","url":"{}""#, json_escape(url)),
+                _ => String::new(),
+            };
+            format!(
+                r#"{{"text":"{}","page":{},"x":{:.2},"y":{:.2},"width":{:.2},"height":{:.2},"font":"{}","font_size":{:.2},"is_bold":{},"is_italic":{},"is_underline":{},"item_type":"{}","mcid":{}{}}}"#,
+                json_escape(&item.text),
+                item.page,
+                item.x,
+                item.y,
+                item.width,
+                item.height,
+                json_escape(&item.font),
+                item.font_size,
+                item.is_bold,
+                item.is_italic,
+                item.is_underline,
+                item_type_label(&item.item_type),
+                mcid,
+                link_url,
+            )
+        })
+        .collect::<Vec<_>>()
+        .join(",");
+
+    format!(
+        r#"{{"total_items":{},"underlined_count":{},"items":[{}]}}"#,
+        items.len(),
+        underlined_count,
+        items_json
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::format_items_json;
+    use pdf_inspector::extractor::ItemType;
+    use pdf_inspector::TextItem;
+
+    #[test]
+    fn items_json_includes_position_and_underline_metadata() {
+        let items = vec![TextItem {
+            text: "A \"quoted\" item".to_string(),
+            x: 12.345,
+            y: 67.891,
+            width: 23.456,
+            height: 9.876,
+            font: "F1".to_string(),
+            font_size: 10.0,
+            page: 2,
+            is_bold: false,
+            is_italic: true,
+            is_underline: true,
+            item_type: ItemType::Text,
+            mcid: Some(7),
+        }];
+
+        let json = format_items_json(&items);
+
+        assert!(json.contains(r#""text":"A \"quoted\" item""#));
+        assert!(json.contains(r#""page":2"#));
+        assert!(json.contains(r#""x":12.35"#));
+        assert!(json.contains(r#""is_underline":true"#));
+        assert!(json.contains(r#""item_type":"text""#));
+        assert!(json.contains(r#""mcid":7"#));
+    }
+}
+
 /// Parse a page specification like "1,3,5-10,20" into a HashSet of page numbers.
 fn parse_page_spec(spec: &str) -> Result<HashSet<u32>, String> {
     let mut pages = HashSet::new();
@@ -104,13 +194,15 @@ fn main() {
     if args.len() < 2 {
         eprintln!("Usage: {} <pdf_file> [output_file]", args[0]);
         eprintln!("       {} <pdf_file> --json", args[0]);
+        eprintln!("       {} <pdf_file> --items-json", args[0]);
         eprintln!("       {} <pdf_file> --raw", args[0]);
         eprintln!();
         eprintln!("Converts PDF to Markdown with smart type detection.");
         eprintln!("Returns early if PDF is scanned (OCR needed).");
         eprintln!();
         eprintln!("Options:");
         eprintln!("  --json              Output result as JSON");
+        eprintln!("  --items-json        Output positioned TextItem JSON");
         eprintln!("  --raw               Output only markdown (no headers)");
         eprintln!("  --pages             Insert page break markers (<!-- Page N -->)");
         eprintln!("  --select-pages N    Only process specified pages (e.g. 1,3,5-10)");
@@ -121,6 +213,7 @@ fn main() {
 
     let pdf_path = &args[1];
     let json_output = args.iter().any(|a| a == "--json");
+    let items_json_output = args.iter().any(|a| a == "--items-json");
     let raw_output = args.iter().any(|a| a == "--raw");
     let page_numbers = args.iter().any(|a| a == "--pages");
     let detect_only = args.iter().any(|a| a == "--detect-only");
@@ -145,6 +238,17 @@ fn main() {
             })
         });
 
+    if items_json_output {
+        match extract_text_with_positions_pages(pdf_path, page_filter.as_ref()) {
+            Ok(items) => println!("{}", format_items_json(&items)),
+            Err(e) => {
+                println!(r#"{{"error":"{}"}}"#, json_escape(&e.to_string()));
+                process::exit(1);
+            }
+        }
+        return;
+    }
+
     let output_file = args
         .get(2)
         .filter(|a| !a.starts_with("--"))