From 9298f14f4a8260a449ab35e5d3422b4452a986d5 Mon Sep 17 00:00:00 2001 From: wanikua Date: Wed, 17 Jun 2026 11:14:28 +0800 Subject: [PATCH 1/4] feat: import Markdown (.md) files as study documents Render an uploaded Markdown file to a clean, text-bearing PDF with pdfkit + marked up front, then send it through the exact same extract -> quality-gate -> store pipeline as any PDF. The viewer, concept tagging, knowledge graph, and all four study tools work on Markdown with no further changes, and the text-coverage gate applies unchanged. - lib/md-to-pdf.ts: server-side Markdown -> PDF renderer (headings, ordered and bullet lists, blockquotes, fenced code, GFM tables, inline emphasis, inline code and links), styled to match scripts/generate-sample-pdfs.ts - app/api/upload/route.ts: detect .md/.markdown uploads and convert before the %PDF- check; empty/oversized markdown returns a friendly 422 - components/UploadCard.tsx: accept Markdown in the file picker, drop zone, validation, and copy - package.json: promote marked from a dev to a runtime dependency, since the converter runs in the packaged app Latin scripts only for now: the standard PDF fonts cover Latin, while CJK and other scripts would need an embedded font (a deliberate follow-up). --- app/api/upload/route.ts | 26 +++ components/UploadCard.tsx | 14 +- lib/md-to-pdf.ts | 435 ++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 4 files changed, 471 insertions(+), 6 deletions(-) create mode 100644 lib/md-to-pdf.ts diff --git a/app/api/upload/route.ts b/app/api/upload/route.ts index afa6b20..682ccc2 100644 --- a/app/api/upload/route.ts +++ b/app/api/upload/route.ts @@ -25,12 +25,20 @@ import { type PdfRejectReason, type PdfQualityStats, } from "@/lib/pdf-extract"; +import { + markdownToPdf, + MarkdownEmptyError, + MarkdownTooLargeError, +} from "@/lib/md-to-pdf"; import { ensureDocDir, pdfPath } from "@/lib/paths"; import { getDoc, newDocId, saveDoc } from "@/lib/store"; export const runtime = "nodejs"; export const maxDuration = 60; +/** Extensions we accept as Markdown and render to PDF before ingesting. */ +const MARKDOWN_EXT = /\.(md|markdown|mdown|mkd|mdwn)$/i; + const SAMPLE_NAME_TO_DOC_ID: Record = { anatomy: "sample-anatomy", physics: "sample-physics", @@ -105,6 +113,24 @@ export async function POST(req: Request) { buffer = Buffer.from(await file.arrayBuffer()); const fname = (file as unknown as { name?: string }).name; if (fname) filename = fname.replace(/[^a-z0-9._-]/gi, "_"); + + // Markdown uploads: render to a clean, text-bearing PDF up front, then + // fall through to the exact same extract → gate → store pipeline as any + // other PDF. The %PDF- sanity check below then validates the rendered + // bytes (pdfkit emits a 1.7 header). + if (MARKDOWN_EXT.test(filename)) { + try { + buffer = await markdownToPdf(buffer.toString("utf-8")); + } catch (e) { + if (e instanceof MarkdownEmptyError || e instanceof MarkdownTooLargeError) { + return NextResponse.json({ error: e.message }, { status: 422 }); + } + return NextResponse.json( + { error: "This Markdown file couldn't be converted to a document." }, + { status: 422 }, + ); + } + } } } else { return NextResponse.json({ error: "expected multipart/form-data" }, { status: 400 }); diff --git a/components/UploadCard.tsx b/components/UploadCard.tsx index 2b41790..3940c84 100644 --- a/components/UploadCard.tsx +++ b/components/UploadCard.tsx @@ -83,6 +83,10 @@ const FILENAME_TO_TITLE: Record = { "chemistry.pdf": "Organic Chemistry", }; +/** A PDF, or a Markdown file we render to PDF on upload. Kept in sync with + * MARKDOWN_EXT in app/api/upload/route.ts. */ +const ACCEPTED_FILE = /\.(pdf|md|markdown|mdown|mkd|mdwn)$/i; + function humaniseAgo(ts: number): string { const dt = Date.now() - ts; if (dt < 5_000) return "just now"; @@ -137,8 +141,8 @@ export default function UploadCard() { const startUpload = useCallback( async (file: File) => { setError(null); - if (!file.name.toLowerCase().endsWith(".pdf")) { - setError("Please pick a PDF file"); + if (!ACCEPTED_FILE.test(file.name)) { + setError("Please pick a PDF or Markdown (.md) file"); return; } setBusy("upload"); @@ -203,7 +207,7 @@ export default function UploadCard() { { const f = e.target.files?.[0]; @@ -237,7 +241,7 @@ export default function UploadCard() { ) : ( <> - Drop your PDF here, or + Drop your PDF or Markdown here, or Select the file @@ -246,7 +250,7 @@ export default function UploadCard() { )}

- Text-tagged PDFs work best. No OCR. + Text-based PDFs and Markdown (.md) work best. No OCR.

diff --git a/lib/md-to-pdf.ts b/lib/md-to-pdf.ts new file mode 100644 index 0000000..85562b6 --- /dev/null +++ b/lib/md-to-pdf.ts @@ -0,0 +1,435 @@ +/** + * Server-side Markdown → PDF rendering. + * + * Get It. is built around a PDF: the viewer renders the document page by + * page and overlays concept tags at real PDF-space coordinates, and every + * agent reads the per-page text that `lib/pdf-extract.ts` pulls out. To let + * students study from a `.md` file without rebuilding any of that, we turn + * the markdown into a clean, text-bearing PDF up front and then feed it + * through the exact same `extractPdf` pipeline as a normal upload. + * + * We tokenize with `marked` (already a project dependency) and lay the + * tokens out with `pdfkit` (the same library `scripts/generate-sample-pdfs.ts` + * uses to mint the bundled sample documents), so this adds no new runtime + * dependency. The standard PDF fonts cover Latin scripts; non-Latin scripts + * (CJK, etc.) would need an embedded font and are a deliberate follow-up. + */ + +import PDFDocument from "pdfkit"; +import { marked, type Token, type Tokens } from "marked"; + +/** + * Reject absurdly large markdown before we spend time rendering it. The + * extracted PDF still has to clear `MAX_PDF_PAGES`, but bailing on the raw + * text first keeps a pathological paste from pinning a CPU. ~1M characters + * is far longer than any real study document. + */ +export const MAX_MARKDOWN_BYTES = 1_000_000; + +export class MarkdownEmptyError extends Error { + constructor() { + super("This Markdown file has no readable text."); + this.name = "MarkdownEmptyError"; + } +} + +export class MarkdownTooLargeError extends Error { + constructor() { + super("This Markdown file is too large to import."); + this.name = "MarkdownTooLargeError"; + } +} + +// ── Page geometry & palette ───────────────────────────────────────────── +// A4 in PDF points, with comfortable reading margins. Mirrors the editorial +// "ink on warm white" look of the bundled samples and the writeup PDF. + +const PAGE = { width: 595.28, height: 841.89 }; +const MARGIN = 64; +const CONTENT_WIDTH = PAGE.width - MARGIN * 2; +const PAGE_BOTTOM = PAGE.height - MARGIN; + +const INK_900 = "#0f172a"; +const INK_700 = "#1e293b"; +const INK_500 = "#64748b"; +const ACCENT = "#4f5ae0"; +const RULE = "#cbd5e1"; +const CODE_BG = "#f3f2ef"; + +const FONT = { + regular: "Helvetica", + bold: "Helvetica-Bold", + italic: "Helvetica-Oblique", + boldItalic: "Helvetica-BoldOblique", + mono: "Courier", +} as const; + +/** Point size per heading depth (h1…h6). */ +const HEADING_SIZE = [22, 17, 14, 12.5, 11.5, 11]; +const BODY_SIZE = 11; +const CODE_SIZE = 9; +const LINE_GAP = 2.5; + +type PDFKitDoc = InstanceType; + +/** Inline run after emphasis/link nesting has been flattened to leaves. */ +type Segment = { + text: string; + bold: boolean; + italic: boolean; + mono: boolean; + link?: string; +}; + +type Style = { bold: boolean; italic: boolean }; + +const BASE_STYLE: Style = { bold: false, italic: false }; + +/** Decode the handful of HTML entities `marked` leaves encoded in token text. */ +function decodeEntities(s: string): string { + return s + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, " "); +} + +/** + * Walk an inline token tree (the `tokens` array on a paragraph, heading, + * list item, …) and flatten it to a flat list of styled leaf runs that + * pdfkit can emit as one continued line. + */ +function flattenInline(tokens: Token[] | undefined, style: Style, out: Segment[]): void { + if (!tokens) return; + for (const token of tokens) { + switch (token.type) { + case "text": { + const t = token as Tokens.Text; + if (t.tokens && t.tokens.length) flattenInline(t.tokens, style, out); + else out.push({ text: decodeEntities(t.text), bold: style.bold, italic: style.italic, mono: false }); + break; + } + case "escape": { + const t = token as Tokens.Escape; + out.push({ text: t.text, bold: style.bold, italic: style.italic, mono: false }); + break; + } + case "strong": + flattenInline((token as Tokens.Strong).tokens, { ...style, bold: true }, out); + break; + case "em": + flattenInline((token as Tokens.Em).tokens, { ...style, italic: true }, out); + break; + case "del": + flattenInline((token as Tokens.Del).tokens, style, out); + break; + case "codespan": { + const t = token as Tokens.Codespan; + out.push({ text: decodeEntities(t.text), bold: style.bold, italic: style.italic, mono: true }); + break; + } + case "link": { + const t = token as Tokens.Link; + const before = out.length; + flattenInline(t.tokens, style, out); + for (let i = before; i < out.length; i++) out[i].link = t.href; + break; + } + case "br": + out.push({ text: "\n", bold: style.bold, italic: style.italic, mono: false }); + break; + case "image": { + // We can't lay out images with the standard pipeline; keep the alt + // text so the document still reads and the detector has something. + const t = token as Tokens.Image; + if (t.text) out.push({ text: t.text, bold: style.bold, italic: style.italic, mono: false }); + break; + } + default: { + const t = token as { text?: string }; + if (t.text) out.push({ text: decodeEntities(t.text), bold: style.bold, italic: style.italic, mono: false }); + } + } + } +} + +function fontFor(seg: Segment): string { + if (seg.mono) return FONT.mono; + if (seg.bold && seg.italic) return FONT.boldItalic; + if (seg.bold) return FONT.bold; + if (seg.italic) return FONT.italic; + return FONT.regular; +} + +type EmitOpts = { + size: number; + color: string; + indent?: number; + width?: number; + paragraphGap?: number; +}; + +/** Emit a run of flattened segments as a single wrapped paragraph. */ +function emitSegments(doc: PDFKitDoc, segs: Segment[], opts: EmitOpts): void { + const width = opts.width ?? CONTENT_WIDTH - (opts.indent ?? 0); + if (segs.length === 0) { + doc.text(" ", { width }); + return; + } + const last = segs.length - 1; + segs.forEach((seg, i) => { + doc + .font(fontFor(seg)) + .fontSize(opts.size) + .fillColor(seg.link ? ACCENT : opts.color); + const textOpts: PDFKit.Mixins.TextOptions = { + continued: i < last, + width, + lineGap: LINE_GAP, + }; + if (i === last) textOpts.paragraphGap = opts.paragraphGap ?? 8; + if (seg.link) { + textOpts.link = seg.link; + textOpts.underline = true; + } + doc.text(seg.text, textOpts); + }); +} + +/** Add a page break before a block if it would otherwise be orphaned. */ +function breakIfTight(doc: PDFKitDoc, needed: number): void { + if (doc.y + needed > PAGE_BOTTOM) doc.addPage(); +} + +// ── Block renderers ───────────────────────────────────────────────────── + +function renderHeading(doc: PDFKitDoc, token: Tokens.Heading): void { + const size = HEADING_SIZE[Math.min(token.depth, HEADING_SIZE.length) - 1]; + breakIfTight(doc, size * 2.4); + doc.moveDown(token.depth <= 2 ? 0.6 : 0.4); + const segs: Segment[] = []; + flattenInline(token.tokens, BASE_STYLE, segs); + // Headings render in a single weight regardless of inline emphasis. + for (const s of segs) s.bold = true; + emitSegments(doc, segs, { size, color: INK_900, paragraphGap: 4 }); + doc.moveDown(0.25); +} + +function renderParagraph(doc: PDFKitDoc, token: Tokens.Paragraph): void { + const segs: Segment[] = []; + flattenInline(token.tokens, BASE_STYLE, segs); + emitSegments(doc, segs, { size: BODY_SIZE, color: INK_700, paragraphGap: 8 }); +} + +function renderList(doc: PDFKitDoc, token: Tokens.List, depth = 0): void { + const indent = 18 + depth * 16; + let index = typeof token.start === "number" ? token.start : 1; + for (const item of token.items) { + const marker = token.ordered ? `${index}.` : "•"; + breakIfTight(doc, BODY_SIZE * 2); + const markerX = MARGIN + indent - 14; + const y = doc.y; + doc.font(FONT.regular).fontSize(BODY_SIZE).fillColor(INK_500).text(marker, markerX, y, { + width: 14, + lineGap: LINE_GAP, + }); + // Render the item's own inline content next to the marker, then recurse + // into any nested blocks (sub-lists) underneath it. + doc.x = MARGIN + indent; + doc.y = y; + const inline: Segment[] = []; + const nested: Tokens.List[] = []; + for (const child of item.tokens) { + if (child.type === "list") nested.push(child as Tokens.List); + else if (child.type === "text") flattenInline((child as Tokens.Text).tokens ?? [child as Token], BASE_STYLE, inline); + else flattenInline([child], BASE_STYLE, inline); + } + emitSegments(doc, inline, { + size: BODY_SIZE, + color: INK_700, + indent, + width: CONTENT_WIDTH - indent, + paragraphGap: 3, + }); + doc.x = MARGIN; + for (const sub of nested) renderList(doc, sub, depth + 1); + index++; + } + doc.moveDown(0.3); +} + +function renderCode(doc: PDFKitDoc, token: Tokens.Code): void { + const code = token.text.replace(/\n+$/, ""); + doc.font(FONT.mono).fontSize(CODE_SIZE); + const innerWidth = CONTENT_WIDTH - 20; + const height = doc.heightOfString(code, { width: innerWidth, lineGap: 2 }); + breakIfTight(doc, height + 16); + const top = doc.y; + doc + .save() + .rect(MARGIN, top, CONTENT_WIDTH, height + 14) + .fill(CODE_BG) + .restore(); + doc + .font(FONT.mono) + .fontSize(CODE_SIZE) + .fillColor(INK_700) + .text(code, MARGIN + 10, top + 7, { width: innerWidth, lineGap: 2 }); + doc.x = MARGIN; + doc.y = top + height + 14; + doc.moveDown(0.5); +} + +function renderBlockquote(doc: PDFKitDoc, token: Tokens.Blockquote): void { + const top = doc.y; + doc.x = MARGIN + 16; + for (const child of token.tokens) renderBlock(doc, child); + const bottom = doc.y; + doc + .save() + .lineWidth(2) + .strokeColor(ACCENT) + .moveTo(MARGIN + 4, top) + .lineTo(MARGIN + 4, bottom) + .stroke() + .restore(); + doc.x = MARGIN; + doc.moveDown(0.3); +} + +function renderTable(doc: PDFKitDoc, token: Tokens.Table): void { + const cols = token.header.length; + if (cols === 0) return; + const colWidth = CONTENT_WIDTH / cols; + const cellText = (cell: Tokens.TableCell): string => { + const segs: Segment[] = []; + flattenInline(cell.tokens, BASE_STYLE, segs); + return segs.map((s) => s.text).join(""); + }; + const drawRow = (cells: Tokens.TableCell[], header: boolean): void => { + const font = header ? FONT.bold : FONT.regular; + const color = header ? INK_500 : INK_700; + doc.font(font).fontSize(header ? 9 : 10); + const heights = cells.map((c) => + doc.heightOfString(cellText(c), { width: colWidth - 12 }), + ); + const rowHeight = Math.max(...heights, 14) + 8; + breakIfTight(doc, rowHeight); + const top = doc.y; + cells.forEach((c, i) => { + doc + .font(font) + .fontSize(header ? 9 : 10) + .fillColor(color) + .text(cellText(c), MARGIN + i * colWidth, top + 4, { width: colWidth - 12 }); + }); + doc.y = top + rowHeight; + doc + .save() + .lineWidth(0.7) + .strokeColor(RULE) + .moveTo(MARGIN, doc.y) + .lineTo(MARGIN + CONTENT_WIDTH, doc.y) + .stroke() + .restore(); + }; + doc.moveDown(0.3); + drawRow(token.header, true); + for (const row of token.rows) drawRow(row, false); + doc.x = MARGIN; + doc.moveDown(0.5); +} + +function renderHr(doc: PDFKitDoc): void { + doc.moveDown(0.4); + doc + .save() + .lineWidth(0.7) + .strokeColor(RULE) + .moveTo(MARGIN, doc.y) + .lineTo(MARGIN + CONTENT_WIDTH, doc.y) + .stroke() + .restore(); + doc.moveDown(0.5); +} + +function renderBlock(doc: PDFKitDoc, token: Token): void { + switch (token.type) { + case "heading": + renderHeading(doc, token as Tokens.Heading); + break; + case "paragraph": + renderParagraph(doc, token as Tokens.Paragraph); + break; + case "list": + renderList(doc, token as Tokens.List); + break; + case "code": + renderCode(doc, token as Tokens.Code); + break; + case "blockquote": + renderBlockquote(doc, token as Tokens.Blockquote); + break; + case "table": + renderTable(doc, token as Tokens.Table); + break; + case "hr": + renderHr(doc); + break; + case "space": + doc.moveDown(0.4); + break; + case "html": + break; // raw HTML is dropped — we render text, not markup + default: { + // Anything else with text (e.g. a bare text block) still gets rendered. + const t = token as { text?: string }; + if (t.text && t.text.trim()) { + doc.font(FONT.regular).fontSize(BODY_SIZE).fillColor(INK_700).text(decodeEntities(t.text), { + width: CONTENT_WIDTH, + lineGap: LINE_GAP, + paragraphGap: 8, + }); + } + } + } +} + +/** + * Render a Markdown string to PDF bytes. The returned buffer is a normal, + * text-bearing PDF that `extractPdf` reads exactly like any other upload. + */ +export async function markdownToPdf(markdown: string): Promise { + if (Buffer.byteLength(markdown, "utf-8") > MAX_MARKDOWN_BYTES) { + throw new MarkdownTooLargeError(); + } + if (!markdown.trim()) { + throw new MarkdownEmptyError(); + } + + const tokens = marked.lexer(markdown, { gfm: true }); + + const doc = new PDFDocument({ + size: "A4", + margins: { top: MARGIN, bottom: MARGIN, left: MARGIN, right: MARGIN }, + info: { Producer: "Get It. Markdown Importer" }, + bufferPages: true, + pdfVersion: "1.7", + lang: "en-US", + }); + + const chunks: Buffer[] = []; + doc.on("data", (c: Buffer) => chunks.push(c)); + const done = new Promise((resolve, reject) => { + doc.on("end", () => resolve(Buffer.concat(chunks))); + doc.on("error", reject); + }); + + doc.x = MARGIN; + for (const token of tokens) renderBlock(doc, token); + + doc.end(); + return done; +} diff --git a/package.json b/package.json index 3f0c9a5..7534123 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "framer-motion": "^12.38.0", "katex": "^0.16.45", "lucide-react": "^1.14.0", + "marked": "^14.1.4", "next": "16.2.6", "pdfjs-dist": "^5.7.284", "pdfkit": "^0.18.0", @@ -64,7 +65,6 @@ "electron-builder": "^25.1.8", "eslint": "^9", "eslint-config-next": "16.2.6", - "marked": "^14.1.4", "pdf-lib": "^1.17.1", "playwright": "^1.59.1", "tailwindcss": "^4", From 9d075b2e215a17f3f41e7c0bc37072acf4448e40 Mon Sep 17 00:00:00 2001 From: wanikua Date: Wed, 17 Jun 2026 11:30:04 +0800 Subject: [PATCH 2/4] feat: render CJK Markdown with a system font When the markdown contains CJK / kana / hangul / fullwidth characters, probe for a system CJK font (PingFang / Hiragino on macOS, Microsoft YaHei / SimSun on Windows, Noto Sans CJK / WenQuanYi on Linux) and render the whole document with it. Those families carry Latin glyphs too, so mixed English/Chinese reads correctly; pdfkit subsets the font, so only the glyphs actually used are embedded. When no CJK font is present we fall back to the Latin fonts. Verified round-trip: a Chinese study note renders, embeds, and is re-extracted by pdf.js as correct Chinese text, clearing the text-coverage gate. --- lib/md-to-pdf.ts | 197 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 159 insertions(+), 38 deletions(-) diff --git a/lib/md-to-pdf.ts b/lib/md-to-pdf.ts index 85562b6..eb30812 100644 --- a/lib/md-to-pdf.ts +++ b/lib/md-to-pdf.ts @@ -11,10 +11,18 @@ * We tokenize with `marked` (already a project dependency) and lay the * tokens out with `pdfkit` (the same library `scripts/generate-sample-pdfs.ts` * uses to mint the bundled sample documents), so this adds no new runtime - * dependency. The standard PDF fonts cover Latin scripts; non-Latin scripts - * (CJK, etc.) would need an embedded font and are a deliberate follow-up. + * dependency. + * + * Scripts: the standard PDF fonts cover Latin. When the document contains + * CJK (or kana / hangul / fullwidth) characters we register a system CJK + * font and render the whole document with it — those fonts carry Latin + * glyphs too, so mixed English/Chinese reads correctly. If no CJK font is + * found on the host we fall back to the Latin fonts and Latin text still + * renders; the text-coverage gate downstream will reject a doc that came out + * blank, which is the right outcome on a system with no CJK font at all. */ +import fs from "node:fs"; import PDFDocument from "pdfkit"; import { marked, type Token, type Tokens } from "marked"; @@ -56,13 +64,27 @@ const ACCENT = "#4f5ae0"; const RULE = "#cbd5e1"; const CODE_BG = "#f3f2ef"; -const FONT = { +/** The four weights + monospace a document is drawn with. Swapped wholesale + * for a registered CJK family when the source contains CJK characters. */ +type Fonts = { + regular: string; + bold: string; + italic: string; + boldItalic: string; + mono: string; +}; + +const LATIN_FONTS: Fonts = { regular: "Helvetica", bold: "Helvetica-Bold", italic: "Helvetica-Oblique", boldItalic: "Helvetica-BoldOblique", mono: "Courier", -} as const; +}; + +/** Matches CJK ideographs, kana, hangul, and CJK/fullwidth punctuation. */ +const CJK_RE = + /[　-ヿ㐀-䶿一-鿿豈-﫿＀-￯가-힯]/; /** Point size per heading depth (h1…h6). */ const HEADING_SIZE = [22, 17, 14, 12.5, 11.5, 11]; @@ -72,6 +94,9 @@ const LINE_GAP = 2.5; type PDFKitDoc = InstanceType; +/** Render context threaded through every block renderer. */ +type Ctx = { doc: PDFKitDoc; fonts: Fonts }; + /** Inline run after emphasis/link nesting has been flattened to leaves. */ type Segment = { text: string; @@ -85,6 +110,94 @@ type Style = { bold: boolean; italic: boolean }; const BASE_STYLE: Style = { bold: false, italic: false }; +// ── CJK font resolution ───────────────────────────────────────────────── + +type CjkFace = { path: string; postscript?: string }; +type CjkPair = { regular: CjkFace; bold: CjkFace }; + +/** Per-platform candidate CJK fonts, in preference order. `postscript` names + * a face inside a `.ttc` collection (omitted for single-face `.ttf`/`.otf`). */ +function cjkCandidates(): CjkPair[] { + switch (process.platform) { + case "darwin": + return [ + { + regular: { path: "/System/Library/Fonts/PingFang.ttc", postscript: "PingFangSC-Regular" }, + bold: { path: "/System/Library/Fonts/PingFang.ttc", postscript: "PingFangSC-Semibold" }, + }, + { + regular: { path: "/System/Library/Fonts/Hiragino Sans GB.ttc", postscript: "HiraginoSansGB-W3" }, + bold: { path: "/System/Library/Fonts/Hiragino Sans GB.ttc", postscript: "HiraginoSansGB-W6" }, + }, + { + regular: { path: "/System/Library/Fonts/Supplemental/Arial Unicode.ttf" }, + bold: { path: "/System/Library/Fonts/Supplemental/Arial Unicode.ttf" }, + }, + ]; + case "win32": + return [ + { + regular: { path: "C:\\Windows\\Fonts\\msyh.ttc", postscript: "MicrosoftYaHei" }, + bold: { path: "C:\\Windows\\Fonts\\msyhbd.ttc", postscript: "MicrosoftYaHei-Bold" }, + }, + { + regular: { path: "C:\\Windows\\Fonts\\simsun.ttc", postscript: "SimSun" }, + bold: { path: "C:\\Windows\\Fonts\\simsun.ttc", postscript: "SimSun" }, + }, + { + regular: { path: "C:\\Windows\\Fonts\\malgun.ttf" }, + bold: { path: "C:\\Windows\\Fonts\\malgunbd.ttf" }, + }, + ]; + default: + return [ + { + regular: { path: "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", postscript: "NotoSansCJKsc-Regular" }, + bold: { path: "/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc", postscript: "NotoSansCJKsc-Bold" }, + }, + { + regular: { path: "/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf", postscript: "NotoSansCJKsc-Regular" }, + bold: { path: "/usr/share/fonts/opentype/noto/NotoSansCJKsc-Bold.otf", postscript: "NotoSansCJKsc-Bold" }, + }, + { + regular: { path: "/usr/share/fonts/truetype/wqy/wqy-microhei.ttc" }, + bold: { path: "/usr/share/fonts/truetype/wqy/wqy-microhei.ttc" }, + }, + ]; + } +} + +/** First candidate whose regular face exists on disk, or null. */ +function resolveCjkFont(): CjkPair | null { + for (const c of cjkCandidates()) { + if (fs.existsSync(c.regular.path)) { + return { regular: c.regular, bold: fs.existsSync(c.bold.path) ? c.bold : c.regular }; + } + } + return null; +} + +/** + * If the markdown needs CJK and a system font is available, register it on + * the document and return a CJK font map; otherwise return the Latin map. + */ +function setUpFonts(doc: PDFKitDoc, markdown: string): Fonts { + if (!CJK_RE.test(markdown)) return LATIN_FONTS; + const face = resolveCjkFont(); + if (!face) return LATIN_FONTS; + try { + doc.registerFont("cjk", face.regular.path, face.regular.postscript); + doc.registerFont("cjk-bold", face.bold.path, face.bold.postscript); + } catch { + return LATIN_FONTS; + } + // CJK has no italic; reuse the upright faces. Code uses the CJK font too so + // CJK inside code blocks still renders (at the cost of true monospacing). + return { regular: "cjk", bold: "cjk-bold", italic: "cjk", boldItalic: "cjk-bold", mono: "cjk" }; +} + +// ── Inline flattening ─────────────────────────────────────────────────── + /** Decode the handful of HTML entities `marked` leaves encoded in token text. */ function decodeEntities(s: string): string { return s @@ -155,12 +268,12 @@ function flattenInline(tokens: Token[] | undefined, style: Style, out: Segment[] } } -function fontFor(seg: Segment): string { - if (seg.mono) return FONT.mono; - if (seg.bold && seg.italic) return FONT.boldItalic; - if (seg.bold) return FONT.bold; - if (seg.italic) return FONT.italic; - return FONT.regular; +function fontFor(seg: Segment, fonts: Fonts): string { + if (seg.mono) return fonts.mono; + if (seg.bold && seg.italic) return fonts.boldItalic; + if (seg.bold) return fonts.bold; + if (seg.italic) return fonts.italic; + return fonts.regular; } type EmitOpts = { @@ -172,7 +285,8 @@ type EmitOpts = { }; /** Emit a run of flattened segments as a single wrapped paragraph. */ -function emitSegments(doc: PDFKitDoc, segs: Segment[], opts: EmitOpts): void { +function emitSegments(ctx: Ctx, segs: Segment[], opts: EmitOpts): void { + const { doc, fonts } = ctx; const width = opts.width ?? CONTENT_WIDTH - (opts.indent ?? 0); if (segs.length === 0) { doc.text(" ", { width }); @@ -181,7 +295,7 @@ function emitSegments(doc: PDFKitDoc, segs: Segment[], opts: EmitOpts): void { const last = segs.length - 1; segs.forEach((seg, i) => { doc - .font(fontFor(seg)) + .font(fontFor(seg, fonts)) .fontSize(opts.size) .fillColor(seg.link ? ACCENT : opts.color); const textOpts: PDFKit.Mixins.TextOptions = { @@ -205,7 +319,8 @@ function breakIfTight(doc: PDFKitDoc, needed: number): void { // ── Block renderers ───────────────────────────────────────────────────── -function renderHeading(doc: PDFKitDoc, token: Tokens.Heading): void { +function renderHeading(ctx: Ctx, token: Tokens.Heading): void { + const { doc } = ctx; const size = HEADING_SIZE[Math.min(token.depth, HEADING_SIZE.length) - 1]; breakIfTight(doc, size * 2.4); doc.moveDown(token.depth <= 2 ? 0.6 : 0.4); @@ -213,17 +328,18 @@ function renderHeading(doc: PDFKitDoc, token: Tokens.Heading): void { flattenInline(token.tokens, BASE_STYLE, segs); // Headings render in a single weight regardless of inline emphasis. for (const s of segs) s.bold = true; - emitSegments(doc, segs, { size, color: INK_900, paragraphGap: 4 }); + emitSegments(ctx, segs, { size, color: INK_900, paragraphGap: 4 }); doc.moveDown(0.25); } -function renderParagraph(doc: PDFKitDoc, token: Tokens.Paragraph): void { +function renderParagraph(ctx: Ctx, token: Tokens.Paragraph): void { const segs: Segment[] = []; flattenInline(token.tokens, BASE_STYLE, segs); - emitSegments(doc, segs, { size: BODY_SIZE, color: INK_700, paragraphGap: 8 }); + emitSegments(ctx, segs, { size: BODY_SIZE, color: INK_700, paragraphGap: 8 }); } -function renderList(doc: PDFKitDoc, token: Tokens.List, depth = 0): void { +function renderList(ctx: Ctx, token: Tokens.List, depth = 0): void { + const { doc, fonts } = ctx; const indent = 18 + depth * 16; let index = typeof token.start === "number" ? token.start : 1; for (const item of token.items) { @@ -231,7 +347,7 @@ function renderList(doc: PDFKitDoc, token: Tokens.List, depth = 0): void { breakIfTight(doc, BODY_SIZE * 2); const markerX = MARGIN + indent - 14; const y = doc.y; - doc.font(FONT.regular).fontSize(BODY_SIZE).fillColor(INK_500).text(marker, markerX, y, { + doc.font(fonts.regular).fontSize(BODY_SIZE).fillColor(INK_500).text(marker, markerX, y, { width: 14, lineGap: LINE_GAP, }); @@ -246,7 +362,7 @@ function renderList(doc: PDFKitDoc, token: Tokens.List, depth = 0): void { else if (child.type === "text") flattenInline((child as Tokens.Text).tokens ?? [child as Token], BASE_STYLE, inline); else flattenInline([child], BASE_STYLE, inline); } - emitSegments(doc, inline, { + emitSegments(ctx, inline, { size: BODY_SIZE, color: INK_700, indent, @@ -254,15 +370,16 @@ function renderList(doc: PDFKitDoc, token: Tokens.List, depth = 0): void { paragraphGap: 3, }); doc.x = MARGIN; - for (const sub of nested) renderList(doc, sub, depth + 1); + for (const sub of nested) renderList(ctx, sub, depth + 1); index++; } doc.moveDown(0.3); } -function renderCode(doc: PDFKitDoc, token: Tokens.Code): void { +function renderCode(ctx: Ctx, token: Tokens.Code): void { + const { doc, fonts } = ctx; const code = token.text.replace(/\n+$/, ""); - doc.font(FONT.mono).fontSize(CODE_SIZE); + doc.font(fonts.mono).fontSize(CODE_SIZE); const innerWidth = CONTENT_WIDTH - 20; const height = doc.heightOfString(code, { width: innerWidth, lineGap: 2 }); breakIfTight(doc, height + 16); @@ -273,7 +390,7 @@ function renderCode(doc: PDFKitDoc, token: Tokens.Code): void { .fill(CODE_BG) .restore(); doc - .font(FONT.mono) + .font(fonts.mono) .fontSize(CODE_SIZE) .fillColor(INK_700) .text(code, MARGIN + 10, top + 7, { width: innerWidth, lineGap: 2 }); @@ -282,10 +399,11 @@ function renderCode(doc: PDFKitDoc, token: Tokens.Code): void { doc.moveDown(0.5); } -function renderBlockquote(doc: PDFKitDoc, token: Tokens.Blockquote): void { +function renderBlockquote(ctx: Ctx, token: Tokens.Blockquote): void { + const { doc } = ctx; const top = doc.y; doc.x = MARGIN + 16; - for (const child of token.tokens) renderBlock(doc, child); + for (const child of token.tokens) renderBlock(ctx, child); const bottom = doc.y; doc .save() @@ -299,7 +417,8 @@ function renderBlockquote(doc: PDFKitDoc, token: Tokens.Blockquote): void { doc.moveDown(0.3); } -function renderTable(doc: PDFKitDoc, token: Tokens.Table): void { +function renderTable(ctx: Ctx, token: Tokens.Table): void { + const { doc, fonts } = ctx; const cols = token.header.length; if (cols === 0) return; const colWidth = CONTENT_WIDTH / cols; @@ -309,7 +428,7 @@ function renderTable(doc: PDFKitDoc, token: Tokens.Table): void { return segs.map((s) => s.text).join(""); }; const drawRow = (cells: Tokens.TableCell[], header: boolean): void => { - const font = header ? FONT.bold : FONT.regular; + const font = header ? fonts.bold : fonts.regular; const color = header ? INK_500 : INK_700; doc.font(font).fontSize(header ? 9 : 10); const heights = cells.map((c) => @@ -355,31 +474,31 @@ function renderHr(doc: PDFKitDoc): void { doc.moveDown(0.5); } -function renderBlock(doc: PDFKitDoc, token: Token): void { +function renderBlock(ctx: Ctx, token: Token): void { switch (token.type) { case "heading": - renderHeading(doc, token as Tokens.Heading); + renderHeading(ctx, token as Tokens.Heading); break; case "paragraph": - renderParagraph(doc, token as Tokens.Paragraph); + renderParagraph(ctx, token as Tokens.Paragraph); break; case "list": - renderList(doc, token as Tokens.List); + renderList(ctx, token as Tokens.List); break; case "code": - renderCode(doc, token as Tokens.Code); + renderCode(ctx, token as Tokens.Code); break; case "blockquote": - renderBlockquote(doc, token as Tokens.Blockquote); + renderBlockquote(ctx, token as Tokens.Blockquote); break; case "table": - renderTable(doc, token as Tokens.Table); + renderTable(ctx, token as Tokens.Table); break; case "hr": - renderHr(doc); + renderHr(ctx.doc); break; case "space": - doc.moveDown(0.4); + ctx.doc.moveDown(0.4); break; case "html": break; // raw HTML is dropped — we render text, not markup @@ -387,7 +506,7 @@ function renderBlock(doc: PDFKitDoc, token: Token): void { // Anything else with text (e.g. a bare text block) still gets rendered. const t = token as { text?: string }; if (t.text && t.text.trim()) { - doc.font(FONT.regular).fontSize(BODY_SIZE).fillColor(INK_700).text(decodeEntities(t.text), { + ctx.doc.font(ctx.fonts.regular).fontSize(BODY_SIZE).fillColor(INK_700).text(decodeEntities(t.text), { width: CONTENT_WIDTH, lineGap: LINE_GAP, paragraphGap: 8, @@ -427,8 +546,10 @@ export async function markdownToPdf(markdown: string): Promise { doc.on("error", reject); }); + const ctx: Ctx = { doc, fonts: setUpFonts(doc, markdown) }; + doc.x = MARGIN; - for (const token of tokens) renderBlock(doc, token); + for (const token of tokens) renderBlock(ctx, token); doc.end(); return done; From 2ffc23234187cf35c2b52330825746d466b435c9 Mon Sep 17 00:00:00 2001 From: wanikua Date: Wed, 17 Jun 2026 12:01:58 +0800 Subject: [PATCH 3/4] test: behavior tests for the Markdown importer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/test-md-import.ts (run via `npm run test:md`), matching the existing tsx + check() harness. Tests the public interface only — markdownToPdf and the bytes it returns, round-tripped through the same extractPdf the upload pipeline uses — so they survive a rewrite of the layout internals: - renders a valid PDF - round-trip fidelity: heading, bold, inline code, list, blockquote, code block, table cell, and link text all survive into the extracted text, and a substantive doc clears the text-coverage gate - empty / whitespace-only input is rejected with MarkdownEmptyError - oversized input is rejected with MarkdownTooLargeError - CJK never throws and yields a valid PDF; the Chinese round-trip is asserted when a CJK system font is present and skipped otherwise (bare CI box) --- package.json | 1 + scripts/test-md-import.ts | 172 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 scripts/test-md-import.ts diff --git a/package.json b/package.json index 7534123..9684fa5 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "lint": "eslint", "generate-pdfs": "tsx scripts/generate-sample-pdfs.ts", "test:errors": "tsx scripts/test-error-handling.ts", + "test:md": "tsx scripts/test-md-import.ts", "smoke": "node scripts/smoke-test.mjs", "smoke-all": "node scripts/full-test.mjs", "build-writeup": "node scripts/build-pdf.mjs", diff --git a/scripts/test-md-import.ts b/scripts/test-md-import.ts new file mode 100644 index 0000000..5435819 --- /dev/null +++ b/scripts/test-md-import.ts @@ -0,0 +1,172 @@ +/** + * Behavior tests for the Markdown importer (lib/md-to-pdf.ts). + * + * Run: npx tsx scripts/test-md-import.ts + * + * These exercise the PUBLIC interface only — `markdownToPdf` and the bytes it + * returns — and verify the document round-trips through the same `extractPdf` + * the upload pipeline uses. They make no assumptions about how the renderer + * lays anything out, so they survive a rewrite of the layout internals: the + * contract is "a .md file becomes a text-bearing PDF whose text the agents can + * read", and that is what we assert. + */ + +import { + markdownToPdf, + MarkdownEmptyError, + MarkdownTooLargeError, + MAX_MARKDOWN_BYTES, +} from "../lib/md-to-pdf"; +import { extractPdf, assessPdfQuality } from "../lib/pdf-extract"; + +let failures = 0; +function check(name: string, cond: boolean, detail?: string) { + if (!cond) failures++; + console.log(`${cond ? "PASS" : "FAIL"} ${name}${detail ? ` — ${detail}` : ""}`); +} +function skip(name: string, detail?: string) { + console.log(`SKIP ${name}${detail ? ` — ${detail}` : ""}`); +} + +/** Collapse whitespace so substring checks ignore the renderer's spacing. */ +const flat = (s: string) => s.replace(/\s+/g, " ").trim(); + +/** Render markdown and pull the text back out the way the pipeline does. */ +async function roundTrip(md: string) { + const pdf = await markdownToPdf(md); + const u8 = new Uint8Array(pdf.byteLength); + u8.set(pdf); + const extracted = await extractPdf(u8); + const text = flat(extracted.pages.map((p) => p.text).join("\n")); + return { pdf, extracted, text }; +} + +async function main() { + // 1) Tracer bullet: a basic document renders to a real PDF. + { + const pdf = await markdownToPdf("# Title\n\nA paragraph of body text."); + check( + "renders a valid PDF (starts with %PDF-)", + pdf.length > 0 && pdf.subarray(0, 5).toString("ascii") === "%PDF-", + `${pdf.length} bytes, header ${pdf.subarray(0, 8).toString("ascii")}`, + ); + } + + // 2) Round-trip fidelity: every block type's text survives into the PDF, so the + // detector / chat / flashcards see the real content, not a blank page. + { + const md = [ + "# Photosynthesis", + "", + "Plants convert **light** energy into _chemical_ energy stored as `glucose`.", + "", + "Photosynthesis is the process by which green plants, algae, and some bacteria", + "capture energy from sunlight and use it to synthesise organic compounds from", + "carbon dioxide and water. It is the foundation of almost every food chain on", + "the planet and the original source of the oxygen in the atmosphere. The overall", + "reaction splits water, releases oxygen as a by-product, and fixes carbon into", + "sugars that store the captured energy in their chemical bonds for later use.", + "", + "## Inputs", + "", + "1. Carbon dioxide from the air.", + "2. Water drawn up through the roots.", + "", + "> The light reactions occur in the thylakoid membrane.", + "", + "```", + "6 CO2 + 6 H2O -> C6H12O6 + 6 O2", + "```", + "", + "| Stage | Location |", + "|-------|----------|", + "| Light reactions | Thylakoid |", + "", + "See [the chapter](https://example.com/photosynthesis) for the full pathway.", + ].join("\n"); + const { text, extracted } = await roundTrip(md); + check("round-trip: heading text present", text.includes("Photosynthesis")); + check("round-trip: bold inline text present", text.includes("light")); + check("round-trip: inline code text present", text.includes("glucose")); + check("round-trip: list item text present", text.includes("Carbon dioxide from the air")); + check("round-trip: blockquote text present", text.includes("thylakoid membrane")); + check("round-trip: code block text present", text.includes("C6H12O6")); + check("round-trip: table cell text present", text.includes("Thylakoid")); + check("round-trip: link label text present", text.includes("the chapter")); + check( + "round-trip: a substantive doc clears the text-coverage gate", + assessPdfQuality(extracted).ok, + JSON.stringify(assessPdfQuality(extracted).stats), + ); +} + +// 3) Empty / whitespace-only markdown is rejected with a typed error. +{ + for (const [label, input] of [ + ["empty string", ""], + ["whitespace only", " \n\t \n"], + ] as const) { + let thrown: unknown; + try { + await markdownToPdf(input); + } catch (e) { + thrown = e; + } + check(`rejects ${label} with MarkdownEmptyError`, thrown instanceof MarkdownEmptyError); + } +} + +// 4) Oversized markdown is rejected before rendering. +{ + const huge = "a".repeat(MAX_MARKDOWN_BYTES + 1); + let thrown: unknown; + try { + await markdownToPdf(huge); + } catch (e) { + thrown = e; + } + check("rejects >MAX_MARKDOWN_BYTES with MarkdownTooLargeError", thrown instanceof MarkdownTooLargeError); +} + +// 5) CJK is always safe to render; when the host has a CJK font, the Chinese +// text round-trips. On a host with no CJK font (e.g. a bare CI box) the +// renderer falls back to Latin fonts — still a valid PDF — so we only assert +// the strong extraction when a CJK font is actually present. +{ + const zh = "光合作用把光能转化为化学能"; + const md = + `# 光合作用\n\n${zh},并以葡萄糖的形式储存在植物体内,这是地球上几乎所有生命能量的最终来源。` + + "光反应发生在类囊体膜上,暗反应也就是卡尔文循环发生在叶绿体基质中,二者协同把二氧化碳固定为有机物。"; + let pdf: Buffer | undefined; + let threw = false; + try { + const r = await roundTrip(md); + pdf = r.pdf; + if (r.text.includes(zh)) { + check("CJK: Chinese text round-trips when a CJK font is present", true); + } else { + skip("CJK: Chinese text round-trips", "no CJK system font on this host — fell back to Latin"); + } + } catch { + threw = true; + } + check( + "CJK: rendering never throws and yields a valid PDF", + !threw && !!pdf && pdf.subarray(0, 5).toString("ascii") === "%PDF-", + ); + } +} + +main() + .then(() => { + console.log(""); + if (failures > 0) { + console.error(`✗ ${failures} check(s) failed`); + process.exit(1); + } + console.log("✓ all checks passed"); + }) + .catch((e) => { + console.error(e); + process.exit(1); + }); From b7989ea259f8490663d74570ded91297f4f51447 Mon Sep 17 00:00:00 2001 From: wanikua Date: Wed, 17 Jun 2026 12:08:37 +0800 Subject: [PATCH 4/4] feat: strip .md/.markdown from document titles too Library, viewer, and the KG job titler stripped only `.pdf` from a filename when no curated title matched, so an imported `notes.md` displayed as `notes.md`. Extend the suffix strip to the markdown extensions we accept, mirroring MARKDOWN_EXT in the upload route. --- app/library/library-client.tsx | 2 +- app/viewer/[docId]/viewer-client.tsx | 2 +- components/UploadCard.tsx | 2 +- lib/jobs.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/library/library-client.tsx b/app/library/library-client.tsx index e736b4e..5c5b01a 100644 --- a/app/library/library-client.tsx +++ b/app/library/library-client.tsx @@ -48,7 +48,7 @@ const FILENAME_TO_TITLE: Record = { }; function titleOf(filename: string): string { - return FILENAME_TO_TITLE[filename] ?? filename.replace(/\.pdf$/i, ""); + return FILENAME_TO_TITLE[filename] ?? filename.replace(/\.(pdf|md|markdown|mdown|mkd|mdwn)$/i, ""); } function humaniseAgo(ts: number): string { diff --git a/app/viewer/[docId]/viewer-client.tsx b/app/viewer/[docId]/viewer-client.tsx index 8ca3166..a9b257b 100644 --- a/app/viewer/[docId]/viewer-client.tsx +++ b/app/viewer/[docId]/viewer-client.tsx @@ -286,7 +286,7 @@ export default function ViewerClient({ docId }: { docId: string }) { const docTitle = useMemo( () => - meta && (FILENAME_TO_TITLE[meta.filename] || meta.filename.replace(/\.pdf$/i, "")), + meta && (FILENAME_TO_TITLE[meta.filename] || meta.filename.replace(/\.(pdf|md|markdown|mdown|mkd|mdwn)$/i, "")), [meta], ); diff --git a/components/UploadCard.tsx b/components/UploadCard.tsx index 3940c84..1a86d8d 100644 --- a/components/UploadCard.tsx +++ b/components/UploadCard.tsx @@ -305,7 +305,7 @@ export default function UploadCard() {
{libraryPreview.map((d) => { const title = - FILENAME_TO_TITLE[d.filename] ?? d.filename.replace(/\.pdf$/i, ""); + FILENAME_TO_TITLE[d.filename] ?? d.filename.replace(/\.(pdf|md|markdown|mdown|mkd|mdwn)$/i, ""); return (