feat(import): Open Knowledge Format (OKF) import

src/cli.ts

@@ -24,6 +24,7 @@ import evalCommand, {
   evalJudgementsCommand,
 } from "./commands/eval.js";
 import exportCommand from "./commands/export.js";
+import importCommand from "./commands/import.js";
 import { schemaInitCommand, schemaShowCommand } from "./commands/schema.js";
 import reviewListCommand from "./commands/review-list.js";
 import reviewShowCommand from "./commands/review-show.js";
@@ -345,6 +346,24 @@ program
     }
   });
 
+program
+  .command("import")
+  .description("Import an OKF bundle as review candidates (default) or live pages (--trusted)")
+  .requiredOption("--okf <dir>", "Path to the OKF bundle directory to import")
+  .option(
+    "--trusted",
+    "Write mapped pages directly into wiki/ instead of staging for review (you vouch for the bundle's contents and its self-declared provenance)",
+  )
+  .option("--dry-run", "Report what would be imported (and skipped) without writing anything")
+  .action(async (options: { okf: string; trusted?: boolean; dryRun?: boolean }) => {
+    try {
+      await importCommand(process.cwd(), options);
+    } catch (err) {
+      console.error(`\x1b[31mError:\x1b[0m ${err instanceof Error ? err.message : err}`);
+      process.exit(1);
+    }
+  });
+
 program
   .command("next")
   .description("Show the recommended next action for this llmwiki project (read-only)")

src/commands/import.ts

@@ -0,0 +1,119 @@
+/**
+ * @file `llmwiki import --okf <dir> [--trusted]`. Default: stage each OKF doc as an
+ * imported review candidate (untrusted external knowledge stays gated behind review).
+ * `--trusted`: write mapped pages straight into wiki/ (validated + collision-checked).
+ */
+import path from "path";
+import { importOkfBundle } from "../import/okf-import.js";
+import { writeCandidate } from "../compiler/candidates.js";
+import { validateWikiPage, atomicWrite } from "../utils/markdown.js";
+import { CONCEPTS_DIR, QUERIES_DIR } from "../utils/constants.js";
+import { refreshAfterImport } from "../import/okf-refresh.js";
+import { acquireLock, releaseLock } from "../utils/lock.js";
+import * as output from "../utils/output.js";
+import type { MappedOkfPage } from "../import/types.js";
+import type { SkippedOkfPage } from "../import/okf-collision.js";
+
+/** CLI options for `import`. */
+export interface ImportOptions { okf?: string; trusted?: boolean; dryRun?: boolean; }
+
+/** Stage one mapped page as an imported review candidate. */
+async function stageCandidate(root: string, page: MappedOkfPage): Promise<void> {
+  await writeCandidate(root, {
+    title: page.title, slug: page.slug, summary: page.summary, sources: page.sources, body: page.body,
+    reviewMode: "imported", heldReasons: [{ code: "imported-okf" }],
+    targetDirectory: page.targetDirectory, okfPath: page.okfPath,
+  });
+}
+
+/**
+ * A mapped page is writable iff it has a non-empty slug and passes page validation.
+ *
+ * The empty-slug guard prevents writing `concepts/.md` for a doc whose path slugifies
+ * to "" (a real title alone passes `validateWikiPage`, which only checks title + body).
+ * `validateWikiPage` is NOT an origin check — imported-origin attribution is guaranteed
+ * upstream by the mapper, which stamps `provenanceState: imported` + an `okf:` source token.
+ */
+export function isWritable(page: MappedOkfPage): boolean {
+  return page.slug.length > 0 && validateWikiPage(page.body);
+}
+
+/** True if writable; warns + returns false otherwise (skip-and-warn, used by both write paths). */
+function validForWrite(page: MappedOkfPage): boolean {
+  if (isWritable(page)) return true;
+  output.status("!", output.warn(`OKF import: ${page.okfPath} (slug "${page.slug}") failed validation; skipped.`));
+  return false;
+}
+
+/**
+ * Write already-validated mapped pages live (--trusted): atomic-write into the target
+ * dir, then refresh. If a write throws mid-loop, the already-written subset is still
+ * refreshed (index/MOC stay consistent) before the error propagates.
+ */
+async function writeTrusted(root: string, pages: MappedOkfPage[]): Promise<void> {
+  const written: string[] = [];
+  try {
+    for (const page of pages) {
+      const dir = page.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
+      await atomicWrite(path.join(root, dir, `${page.slug}.md`), page.body);
+      written.push(page.slug);
+    }
+  } finally {
+    if (written.length) await refreshAfterImport(root, written);
+  }
+}
+
+/** Print the per-page breakdown of a dry-run: what would be written, what's invalid, what collides. */
+function reportPreview(pages: MappedOkfPage[], skipped: SkippedOkfPage[]): void {
+  for (const p of pages) {
+    if (isWritable(p)) {
+      output.status("+", `${p.slug} (${p.targetDirectory}) ← ${p.okfPath}`);
+    } else {
+      output.status("!", output.warn(`invalid (empty slug/title/body): ${p.okfPath}`));
+    }
+  }
+  for (const s of skipped) output.status("!", output.warn(`skip ${s.okfPath} — ${s.reason}`));
+}
+
+/** Report what an import WOULD stage/write and what it would skip, without mutating anything (read-only, no lock). */
+async function previewImport(root: string, okf: string): Promise<void> {
+  const { pages, skipped } = await importOkfBundle(okf, root);
+  const writable = pages.filter(isWritable).length;
+  const dropped = skipped.length + (pages.length - writable);
+  output.status("i", output.dim(`Dry run — would import ${writable} page(s); skip ${dropped}.`));
+  reportPreview(pages, skipped);
+}
+
+/**
+ * Import an OKF bundle into the current project.
+ *
+ * The whole operation runs under `.llmwiki/lock`: BOTH the default staging path
+ * (collision-read + candidate list→write→dedup canonicalization) and `--trusted`
+ * (collision-read + live write + index/MOC refresh) are durable read-modify-writes
+ * that must not race a concurrent `compile`/`approve`. `--dry-run` is read-only and
+ * takes no lock.
+ */
+export default async function importCommand(root: string, options: ImportOptions): Promise<void> {
+  if (!options.okf) throw new Error("import: --okf <dir> is required");
+  if (options.dryRun) {
+    await previewImport(root, options.okf);
+    return;
+  }
+  const locked = await acquireLock(root);
+  if (!locked) {
+    output.status("!", output.error("Could not acquire lock. Try again later."));
+    process.exitCode = 1;
+    return;
+  }
+  try {
+    const { pages, skipped } = await importOkfBundle(options.okf, root);
+    const valid = pages.filter(validForWrite);
+    if (options.trusted) await writeTrusted(root, valid);
+    else { for (const p of valid) await stageCandidate(root, p); }
+    const verb = options.trusted ? "wrote" : "staged";
+    const dropped = skipped.length + (pages.length - valid.length);
+    output.status("+", output.success(`OKF import: ${verb} ${valid.length} page(s); skipped ${dropped}.`));
+  } finally {
+    await releaseLock(root);
+  }
+}

src/commands/review-approve.ts

@@ -24,7 +24,7 @@ import { generateMOC } from "../compiler/obsidian.js";
 import { resolveLinks } from "../compiler/resolver.js";
 import { updateEmbeddings } from "../utils/embeddings.js";
 import { readState, updateSourceState } from "../utils/state.js";
-import { CONCEPTS_DIR } from "../utils/constants.js";
+import { CONCEPTS_DIR, QUERIES_DIR } from "../utils/constants.js";
 import * as output from "../utils/output.js";
 import type { ReviewCandidate } from "../utils/types.js";
 import { runReviewUnderLock, readCandidateUnderLock } from "./review-helpers.js";
@@ -51,7 +51,8 @@ async function approveUnderLock(root: string, id: string): Promise<void> {
     return;
   }
 
-  const pagePath = path.join(root, CONCEPTS_DIR, `${candidate.slug}.md`);
+  const dir = candidate.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
+  const pagePath = path.join(root, dir, `${candidate.slug}.md`);
   await atomicWrite(pagePath, candidate.body);
   output.status("+", output.success(`Approved → ${output.source(pagePath)}`));
 

src/commands/review-show.ts

@@ -19,6 +19,9 @@ function candidateReasons(candidate: ReviewCandidate): string[] {
 function printReviewMetadata(candidate: ReviewCandidate): void {
   output.status("i", output.dim(`review:    ${candidate.reviewMode}`));
   output.status("i", output.dim(`reasons:   ${candidateReasons(candidate).join(", ")}`));
+  if (candidate.okfPath) {
+    output.status("i", output.dim(`okfPath:   ${candidate.okfPath}`));
+  }
   if (candidate.confidence !== undefined) {
     output.status("i", output.dim(`confidence: ${candidate.confidence}`));
   }
@@ -27,6 +30,23 @@ function printReviewMetadata(candidate: ReviewCandidate): void {
   }
 }
 
+/**
+ * Warn the reviewer that an imported candidate's body is untrusted external content.
+ * The human review gate is the primary defense for OKF bundles, so the affordance
+ * to recognize an imported (un-sanitized, possibly prompt-injecting) page must be loud.
+ */
+function printImportedWarning(candidate: ReviewCandidate): void {
+  if (candidate.reviewMode !== "imported") return;
+  output.status(
+    "!",
+    output.warn(
+      "Imported from an external OKF bundle — treat the body as UNTRUSTED content " +
+        "(possible prompt injection); provenance is unverified. " +
+        "Review the full body before approving.",
+    ),
+  );
+}
+
 /** Print a single candidate's full content to stdout. */
 export default async function reviewShowCommand(id: string): Promise<void> {
   const candidate = await loadCandidateOrFail(process.cwd(), id);
@@ -39,6 +59,7 @@ export default async function reviewShowCommand(id: string): Promise<void> {
   output.status("i", output.dim(`sources:    ${candidate.sources.join(", ")}`));
   output.status("i", output.dim(`generated:  ${candidate.generatedAt}`));
   printReviewMetadata(candidate);
+  printImportedWarning(candidate);
 
   console.log();
   console.log(candidate.body);

src/compiler/candidates.ts

@@ -65,6 +65,13 @@ interface CandidateDraft {
   reviewMode?: ReviewMode;
   /** Structured reasons for holding this candidate. */
   heldReasons?: HeldReason[];
+  /**
+   * Wiki subdir the approved page is written to; defaults to concepts.
+   * OKF query docs set `queries` to round-trip back into the right subdir.
+   */
+  targetDirectory?: "concepts" | "queries";
+  /** Original OKF bundle-relative path, for imported candidates. */
+  okfPath?: string;
   /** Confidence parsed from the generated page frontmatter, for display. */
   confidence?: number;
   /** True when the generated page frontmatter declares contradictions. */
@@ -75,7 +82,7 @@ interface CandidateDraft {
 const DEFAULT_HELD_REASONS: HeldReason[] = [{ code: "manual-review-requested" }];
 
 /** All valid ReviewMode values. */
-const VALID_REVIEW_MODES: ReviewMode[] = ["policy", "forced"];
+const VALID_REVIEW_MODES: ReviewMode[] = ["policy", "forced", "imported"];
 
 /** All valid HeldReasonCode values — mirrors the union in policy.ts. */
 const VALID_HELD_REASON_CODES: HeldReasonCode[] = [
@@ -85,6 +92,7 @@ const VALID_HELD_REASON_CODES: HeldReasonCode[] = [
   "provenance-violating",
   "all",
   "manual-review-requested",
+  "imported-okf",
 ];
 
 /** Build a deterministic-but-unique id from a slug and a short random suffix. */
@@ -134,6 +142,8 @@ export async function writeCandidate(
     ...(draft.provenanceViolations ? { provenanceViolations: draft.provenanceViolations } : {}),
     ...(draft.confidence !== undefined ? { confidence: draft.confidence } : {}),
     ...(draft.contradicted !== undefined ? { contradicted: draft.contradicted } : {}),
+    ...(draft.targetDirectory ? { targetDirectory: draft.targetDirectory } : {}),
+    ...(draft.okfPath ? { okfPath: draft.okfPath } : {}),
   };
 
   await atomicWrite(candidatePath(root, candidate.id), JSON.stringify(candidate, null, 2));

src/export/collect.ts

@@ -51,9 +51,15 @@ function readAdvisoryConfidence(meta: Record<string, unknown>): number | undefin
 }
 
 /** Validate and return a ProvenanceState from frontmatter, or undefined. */
-function readProvenanceState(meta: Record<string, unknown>): ProvenanceState | undefined {
+export function readProvenanceState(meta: Record<string, unknown>): ProvenanceState | undefined {
   const value = meta.provenanceState;
-  if (value === "extracted" || value === "merged" || value === "inferred" || value === "ambiguous") {
+  if (
+    value === "extracted" ||
+    value === "merged" ||
+    value === "inferred" ||
+    value === "ambiguous" ||
+    value === "imported"
+  ) {
     return value;
   }
   return undefined;

src/import/okf-collision.ts

@@ -0,0 +1,41 @@
+/**
+ * @file Collision policy for OKF import (v1: skip + warn, never overwrite). A mapped
+ * slug clashes with an existing live page, a pending review candidate, or an earlier
+ * doc in the same import. First (path-sorted) wins; later clashes are dropped.
+ */
+import { access } from "fs/promises";
+import path from "path";
+import { CONCEPTS_DIR, QUERIES_DIR } from "../utils/constants.js";
+import { listCandidates } from "../compiler/candidates.js";
+import * as output from "../utils/output.js";
+import type { MappedOkfPage } from "./types.js";
+
+/** A dropped doc + why, for caller reporting. */
+export interface SkippedOkfPage { slug: string; okfPath: string; reason: "live-page" | "pending-candidate" | "duplicate-in-bundle"; }
+
+async function fileExists(p: string): Promise<boolean> {
+  try { await access(p); return true; } catch { return false; }
+}
+
+/** Partition mapped pages into those safe to import and those skipped for a collision. */
+export async function filterCollisions(
+  root: string,
+  pages: MappedOkfPage[],
+): Promise<{ kept: MappedOkfPage[]; skipped: SkippedOkfPage[] }> {
+  const pendingSlugs = new Set((await listCandidates(root)).map((c) => c.slug));
+  const claimed = new Set<string>();
+  const kept: MappedOkfPage[] = [];
+  const skipped: SkippedOkfPage[] = [];
+  for (const page of pages) {
+    let reason: SkippedOkfPage["reason"] | null = null;
+    if (claimed.has(page.slug)) reason = "duplicate-in-bundle";
+    else if (pendingSlugs.has(page.slug)) reason = "pending-candidate";
+    else if (await fileExists(path.join(root, CONCEPTS_DIR, `${page.slug}.md`)) ||
+             await fileExists(path.join(root, QUERIES_DIR, `${page.slug}.md`))) reason = "live-page";
+    if (reason) {
+      output.status("!", output.warn(`OKF import: skipped ${page.okfPath} (slug "${page.slug}" collides: ${reason})`));
+      skipped.push({ slug: page.slug, okfPath: page.okfPath, reason });
+    } else { claimed.add(page.slug); kept.push(page); }
+  }
+  return { kept, skipped };
+}

src/import/okf-import.ts

@@ -0,0 +1,40 @@
+/**
+ * @file Orchestrate an OKF import: read the bundle (confined + bounded), invert each
+ * doc to a llmwiki page, then apply the collision policy. Side-effect free with respect
+ * to wiki/ — the caller (command) decides to stage as candidates or write live.
+ */
+import path from "path";
+import { slugify } from "../utils/markdown.js";
+import { readOkfBundle } from "./okf-read.js";
+import { okfDocToPage, slugFromRelPath } from "./okf-map.js";
+import { filterCollisions } from "./okf-collision.js";
+import type { OkfImportLimits, MappedOkfPage, RawOkfDoc } from "./types.js";
+import type { SkippedOkfPage } from "./okf-collision.js";
+
+/** Stable, filesystem-safe bundle id derived from the bundle directory name. */
+function bundleIdFor(bundleDir: string): string {
+  return slugify(path.basename(path.resolve(bundleDir))) || "bundle";
+}
+
+/** Build a slug->title resolver over the whole bundle so intra-bundle links reverse cleanly. */
+function titleResolver(docs: RawOkfDoc[]): (slug: string) => string | null {
+  const map = new Map<string, string>();
+  for (const d of docs) {
+    const slug = slugFromRelPath(d.relPath);
+    if (typeof d.meta.title === "string") map.set(slug, d.meta.title);
+  }
+  return (slug) => map.get(slug) ?? null;
+}
+
+/** Read + map + collision-filter an OKF bundle into stage-ready pages. */
+export async function importOkfBundle(
+  bundleDir: string,
+  root: string,
+  overrides: Partial<OkfImportLimits> = {},
+): Promise<{ pages: MappedOkfPage[]; skipped: SkippedOkfPage[] }> {
+  const docs = await readOkfBundle(bundleDir, overrides);
+  const ctx = { bundleId: bundleIdFor(bundleDir), titleOf: titleResolver(docs) };
+  const mapped = docs.map((d) => okfDocToPage(d, ctx));
+  const { kept, skipped } = await filterCollisions(root, mapped);
+  return { pages: kept, skipped };
+}

src/import/okf-limits.ts

@@ -0,0 +1,19 @@
+/**
+ * @file Resource caps bounding untrusted OKF bundle ingestion (DoS guard).
+ *
+ * Note: imported frontmatter is bounded by `maxDocBytes` (the whole-doc cap) only —
+ * v1 has no separate frontmatter byte/depth limit. This is acceptable because
+ * js-yaml's default schema resolves aliases by shared reference (no billion-laughs
+ * amplification) and `buildFrontmatter` re-emits frontmatter via `yaml.dump`.
+ */
+import type { OkfImportLimits } from "./types.js";
+
+/** Conservative defaults; a malformed/hostile bundle is rejected, never processed unboundedly. */
+export const DEFAULT_OKF_LIMITS: OkfImportLimits = {
+  maxFiles: 5000,
+  maxDocBytes: 2_000_000,
+  maxTotalBytes: 100_000_000,
+  // Total directory entries visited during the walk — bounds deep-empty-dir /
+  // many-non-`.md` trees that wouldn't otherwise count toward maxFiles.
+  maxEntries: 100_000,
+};