.pre-commit-config.yaml

# MLSysBook Pre-commit Configuration
# =============================================================================
# Single source of truth: every book-* check dispatches through `./book/binder`.
# That means one entry point, one error format, one place to add new checks.
#
# Hook naming mirrors the binder command tree. To find what a hook does, run
# its binder command directly:
#
#     book-check-refs        →  ./book/binder check refs
#     book-check-prose       →  ./book/binder check prose
#     book-check-footnotes   →  ./book/binder check footnotes
#     book-check-cli-contract → ./book/binder check cli
#     book-format-python     →  ./book/binder format python
#
# Each `binder check <group>` invocation runs only the curated default scopes
# (Scope(default=True) in book/cli/commands/validate.py). Heavier / opt-in
# scopes are reachable via `--scope <name>` or `--all-scopes` on the CLI.
#
# Adding a new check
# ------------------
# 1. Implement the runner inside `ValidateCommand` (book/cli/commands/validate.py).
# 2. Add a `Scope("<name>", "_run_<scope>", default=True)` to the right group.
# 3. That's it — pre-commit picks it up automatically because the existing
#    `book-check-<group>` hook already dispatches the new scope.
# 4. Only add a NEW pre-commit hook if you need a brand-new group, or you need
#    to pass scope-specific flags (see book-check-labels / book-check-headers).
#
# Install:           pip install pre-commit && pre-commit install
# Run all hooks:     pre-commit run --all-files
# Run one hook:      pre-commit run <hook-id> --all-files
# Run manual hook:   pre-commit run --hook-stage manual <hook-id>
#
# Sections (in execution order):
#   1. GENERIC          — third-party file-level hygiene (whitespace, JSON, ...)
#   2. BIBLIOGRAPHY     — .bib mechanical fix → tidy → semantic lint
#   3. REPO GUARDS      — cross-cutting checks (mirror drift, internal links)
#   4. BOOK FORMATTERS  — auto-fix mutators (mdformat + binder format ...)
#   5. BOOK VALIDATORS  — one hook per binder check group
#   6. MANUAL           — heavy / on-demand-only hooks
#   7. CI HYGIENE       — workflow safety
#   8. VAULT / STAFFML  — vault-cli schema + ruff
#
# History of retired hooks lives in `.pre-commit-history.md` (next to this file).
# =============================================================================

repos:
  # ===========================================================================
  # SECTION 1: GENERIC — third-party file-level hygiene
  # ===========================================================================

  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v5.0.0
    hooks:
      - id: trailing-whitespace
        name: "Generic: Trim trailing whitespace"
        # Only on markdown to avoid breaking templates / extensions / fixtures.
        files: "\\.(md|qmd)$"
        # site/newsletter/posts/ is auto-synced from Buttondown and uses intentional
        # 2-space markdown hard line breaks; exempt it from whitespace normalization.
        exclude: "^(_site/|_book/|node_modules/|site/newsletter/posts/)"
      - id: end-of-file-fixer
        name: "Generic: Fix end-of-file newlines"
        files: "\\.(md|qmd)$"
        exclude: "^(_site/|_book/|node_modules/|site/newsletter/posts/)"
      - id: check-json
        name: "Generic: Validate JSON syntax"
        # .vscode/*.json allows JSONC (comments); strict JSON does not.
        exclude: "^\\.vscode/"
      - id: check-yaml
        name: "Generic: Validate YAML syntax"
        exclude: "^(_site/)"
      - id: check-merge-conflict
        name: "Generic: Block unresolved merge-conflict markers"
      - id: check-added-large-files
        name: "Generic: Block large files (>5 MB)"
        # 5 MB is comfortably larger than normal PNG/SVG figures but
        # catches datasets, model weights, and video assets that don't
        # belong in the repo. Override per-hook for legitimate large
        # assets if necessary.
        args: ["--maxkb=5120"]
      - id: detect-private-key
        name: "Generic: Detect committed private keys"
      - id: check-case-conflict
        name: "Generic: Block filename case collisions (macOS/Linux)"

  - repo: https://github.com/codespell-project/codespell
    # Must match .github/workflows/codespell.yml: `codespell --toml pyproject.toml`.
    rev: v2.4.1
    hooks:
      - id: codespell
        name: "Generic: Check for common misspellings"
        args: ["--toml", "pyproject.toml"]
        exclude: "^(_site/|_book/|htmlcov/|site/assets/games/vendor/|.*\\.js$|.*\\.pdf$)"

  # ===========================================================================
  # SECTION 2: BIBLIOGRAPHY — .bib mechanical fix → tidy → semantic lint
  # Order matters. Mechanical fixes are idempotent (rewrite if needed, exit 1
  # so the dev re-stages). Tidy reformats. Lint checks semantics last so it
  # validates the post-tidy state.
  # ===========================================================================

  - repo: local
    hooks:
      - id: bib-apply-mechanical
        name: "Bib: Apply safe §5 mechanical fixes"
        # Same logic as the first step of `book/binder bib normalize`.
        # Exits 1 if it rewrote a file (re-stage and commit again).
        entry: ./book/binder bib mechanical --pre-commit
        language: system
        pass_filenames: true
        files: \.bib$

  - repo: https://github.com/FlamingTempura/bibtex-tidy
    rev: v1.14.0
    hooks:
      - id: bibtex-tidy
        name: "Bib: Tidy bibtex files (format)"
        args: [
          "--align=space", "--curly", "--sort=key", "--sort-fields",
          "--duplicates=key", "--remove-empty-fields", "--space=2",
          "--trailing-commas", "--escape", "--wrap=100", "--blank-lines",
        ]
        files: \.bib$

  - repo: local
    hooks:
      - id: book-check-bib
        name: "Bib: Validate against §5 (semantic)"
        # Runs binder check bib → curated scopes (hygiene + style ratchet +
        # volume-scoped citation integrity). The key-content and orphan scopes are
        # default=False until legacy keys/orphans are cleaned up; opt in via
        # `./book/binder check bib --all-scopes`.
        # Baseline allow-lists under book/tools/ grandfather pre-existing
        # hard errors and warning/info metadata debt; only NEW issues block.
        entry: ./book/binder check bib
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.(bib|qmd)$

  # ===========================================================================
  # SECTION 3: REPO GUARDS — cross-cutting checks not tied to one binder group
  # ===========================================================================

  - repo: local
    hooks:
      - id: check-shared-mirrors
        name: "Repo: Verify shared asset mirrors are in sync"
        # Quarto's resource-copy step preserves symlinks instead of dereferencing
        # them, so we keep real-file copies of certain shared assets per subsite.
        # Without this guard the canonical and the mirrors silently diverge — most
        # commonly the wrong subscribe modal renders on one subsite. Always-run
        # because a mirror can drift via deletion of the canonical.
        entry: bash shared/scripts/sync-mirrors.sh --check
        language: system
        pass_filenames: false
        always_run: true

      - id: check-internal-links
        name: "Repo: Validate internal markdown links + anchors"
        # Tier 1 of the release-prep link strategy. Validates relative-path
        # markdown links and same-file anchors in changed .md/.qmd files.
        # External (http/https) links are NOT checked here — that's Lychee's
        # job in CI (Tier 2 per-site validate, Tier 3 nightly rot scan).
        # See shared/scripts/check-internal-links.py for scope and rationale.
        entry: python3 shared/scripts/check-internal-links.py
        language: system
        pass_filenames: true
        files: \.(md|qmd)$
        # Pre-existing baselines we accept without churn. Tighten as cleaned up.
        exclude: |
          (?x)^(
            _site/|
            _book/|
            node_modules/|
            mlsysim/docs/api/|              # auto-generated quartodoc API stubs
            slides/tinyml/README-edx-original\.md$|  # legacy edX export, archive
            tinytorch/tests/.*README\.md$|  # GitHub line-range anchors (#L74-L99)
            labs/index\.qmd$|               # Volume-N lab dirs are forward-looking
            labs/PROTOCOL\.md$|             # Internal authoring doc, links into ../.claude/
            labs/TEMPLATE\.md$              # Internal authoring doc, links into ../.claude/
          )

  # ===========================================================================
  # SECTION 4: BOOK FORMATTERS — auto-fixers run BEFORE validators so
  # downstream checks see the formatted source.
  # ===========================================================================

  - repo: https://github.com/executablebooks/mdformat
    rev: 0.7.9
    hooks:
      - id: mdformat
        name: "Book: Format quarto markdown"
        additional_dependencies: [mdformat-frontmatter]
        files: ^book/quarto/contents/.*\.qmd$
        pass_filenames: true

  - repo: local
    hooks:
      # Order: structure → whitespace → lists → tables.
      - id: book-format-python
        name: "Book: Format hidden Python cells (Black; executable cells 150)"
        entry: ./book/binder format python
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-format-blanks
        name: "Book: Collapse extra blank lines"
        entry: ./book/binder format blanks
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-format-lists
        name: "Book: Fix list spacing (blank before lists, tight items)"
        entry: ./book/binder format lists
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-format-percent-tables
        name: "Book: Rewrite 'percent' → % inside pipe tables"
        # Tables invert the prose convention (the % symbol, not the word).
        # Runs BEFORE prettify so prettify re-aligns the columns that this
        # substitution shortens. Shares its logic with the percent-in-tables
        # validator scope via cli/checks/percent_tables.py.
        entry: ./book/binder format percent-tables
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-format-mitpress-terms
        name: "Book: Apply §10.7 canonical spellings (trade-off, Wi-Fi, …)"
        # MIT Press canonical spelling dictionary. Runs before prettify so
        # prettify re-aligns columns that term substitutions widen/shorten.
        # Shares logic with the spelling-dict validator scope via
        # cli/checks/mitpress_terms.py; skips glossary files.
        entry: ./book/binder format mitpress-terms
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-format-tables-prettify
        name: "Book: Prettify pipe tables (align columns)"
        entry: ./book/binder format prettify
        language: system
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

  # ===========================================================================
  # SECTION 5: BOOK VALIDATORS — one hook per binder check group.
  # Names map 1:1 to `./book/binder check <group>`. Inside each group, the
  # curated default scopes run. To see a group's scopes:
  #   ./book/binder check <group> help
  # To see what's runnable but opt-in for that group:
  #   ./book/binder check <group> --all-scopes
  #
  # Order is execution order: structural / graph checks first, prose / style
  # last. Failures earlier signal "your structure is broken, fix that before
  # debating em-dashes."
  # ===========================================================================

  - repo: local
    hooks:
      # ---- CLI surface ------------------------------------------------

      - id: book-check-cli-contract
        name: "Book: check Binder CLI command contract"
        # Always-run because this guards the public command/help surface, not
        # content files. It is read-only and does not invoke builds.
        entry: ./book/binder check cli
        language: system
        pass_filenames: false
        always_run: true

      # ---- Structural / cross-file ------------------------------------

      - id: book-check-headers
        name: "Book: check headers (section IDs + headline case)"
        entry: ./book/binder check headers
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-structure
        name: "Book: check structure (heading levels, H2 landings, parts, Purpose)"
        entry: ./book/binder check structure
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      # `labels` is split by scope:
      #   orphans     — both vols (10 vol2 orphans were resolved 2026-05;
      #                  any new ones should fail the gate immediately so
      #                  authors wire up labels at the same time they
      #                  introduce them).
      #   duplicates  — both vols, with the binder default of
      #                  figures+tables+listings only (sections /
      #                  equations have legitimate dup-keys in vol2 WIP).
      - id: book-check-labels-orphans
        name: "Book: check labels --scope orphans (both vols)"
        entry: ./book/binder check labels --scope orphans
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-labels-duplicates
        name: "Book: check labels --scope duplicates (both vols, fig+tbl+lst)"
        entry: ./book/binder check labels --scope duplicates
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-refs
        name: "Book: check refs (cross-refs, citations, inline)"
        entry: ./book/binder check refs
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-footnotes
        name: "Book: check footnotes (definition-shape + placement + integrity + capitalization)"
        entry: ./book/binder check footnotes
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-figures
        name: "Book: check figures (captions, div-syntax, files, alt-text, label-required, margin-captions)"
        # label-required scope grandfathers existing violations via
        # book/tools/audit/baselines/captions_baseline.json (shared with
        # tables.caption-required and listings.caption-required).
        entry: ./book/binder check figures
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$|^book/tools/audit/baselines/captions_baseline\.json$

      - id: book-check-images
        name: "Book: check images (formats, externals, SVG XML)"
        entry: ./book/binder check images
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.(qmd|png|jpg|jpeg|gif|svg)$

      - id: book-check-tables
        name: "Book: check tables (grid → pipe, content hygiene, caption-required)"
        # caption-required scope grandfathers existing violations via
        # book/tools/audit/baselines/captions_baseline.json. Regenerate
        # with book/tools/scripts/maintenance/regen_captions_baseline.py.
        entry: ./book/binder check tables
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$|^book/tools/audit/baselines/captions_baseline\.json$

      - id: book-check-listings
        name: "Book: check listings (#lst- divs carry lst-cap)"
        # New group as of 2026-05-15. caption-required ensures any
        # ::: {#lst-X ...} div has an lst-cap="..." attribute so the
        # listing can be captioned and cross-referenced with @lst-X.
        entry: ./book/binder check listings
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$|^book/tools/audit/baselines/captions_baseline\.json$

      - id: book-check-tables-format
        name: "Book: check table formatting"
        # Different binder command (format with --check), separate from the
        # check tables group above. The check tables group inspects content
        # (bare pipes, fracs, HTML entities); this verifies the FORMATTING
        # state matches what `format tables` would produce.
        entry: ./book/binder format tables --check
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      # ---- Markup / code -----------------------------------------------

      - id: book-check-markup
        name: "Book: check markup (patterns, div-fences, dropcaps)"
        entry: ./book/binder check markup
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-code
        name: "Book: check code (python-echo, _str LaTeX leak, LEGO dead code + units)"
        # Runs the curated default scopes of `binder check code`, which include
        # lego-dead-code AND lego-units (default=yes; _run_lego_units wraps
        # book/tools/scripts/lint_lego_units.py with --fail-on warning, and a
        # run fails on ANY issue regardless of severity — so warnings block the
        # commit exactly as a standalone hook would). There is deliberately no
        # separate book-check-lego-units hook: it would call the same linter a
        # second time, off-Binder. See .pre-commit-history.md (2026-06-10).
        entry: ./book/binder check code
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      # ---- Prose style -------------------------------------------------

      - id: book-check-prose
        name: "Book: check prose (contractions, dup words, above/below, ...)"
        entry: ./book/binder check prose
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-punctuation
        name: "Book: check punctuation (em-dash, slash, vs., e.g., en-dash)"
        entry: ./book/binder check punctuation
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-numbers
        name: "Book: check numbers (units, percent, binary units, currency)"
        entry: ./book/binder check numbers
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.(qmd|json|svg)$|^book/cli/checks/currency_style\.py$|^book/cli/commands/validate\.py$

      - id: book-check-math
        name: "Book: check math (spacing, attr-leaks, fmt/suffix canonical)"
        # Includes canonical scope (LEGO fmt-family + _str/_math suffix discipline).
        # render-audit scope is default=False (10 min HTML build). Run
        # manually via book-check-math-render-audit (manual stage).
        entry: ./book/binder check math
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      # ---- Domain-specific --------------------------------------------

      - id: book-check-notation
        name: "Book: check notation (iron-law symbol consistency)"
        entry: ./book/binder check notation
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-index
        name: "Book: check index (placement + anti-patterns + xrefs + forbidden contexts)"
        # Includes 3 scopes migrated from book/tools/audit/index/* (2026-05-06):
        # anti-patterns, tag-placement, xref-resolves. Plus the original
        # placement scope. The audit scripts remain callable standalone.
        # Added 2026-05-17: forbidden-contexts catches \index{} inside Python
        # code fences, $..$/$$..$$ math, and fig-cap/fig-alt/tbl-cap/lst-cap/
        # title= attribute strings (audit.checks.index_placement).
        entry: ./book/binder check index
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-sources
        name: "Book: check source-note formatting"
        entry: ./book/binder check sources
        language: system
        # The ONE binder-check hook that consumes the changed-file list:
        # _run_sources (validate.py) scopes to ns.files when given them, so we
        # keep pass_filenames: true (explicit, not the inherited default — do
        # not "fix" this to false or the per-file scoping breaks). Anchored to
        # book/quarto/contents/ like every sibling; the previous unanchored
        # \.qmd$ fired on the 250+ qmd files outside contents (instructors/,
        # kits/, slides/). The old validate.py / source_citations.py triggers
        # were dropped: with pass_filenames: true they made the hook check the
        # .py file itself (a silent no-op), defeating the "re-run on checker
        # change" intent rather than serving it.
        pass_filenames: true
        files: ^book/quarto/contents/.*\.qmd$

      # NOTE: no book-check-json hook. JSON *syntax* validation is generic file
      # hygiene owned by the third-party `check-json` hook in SECTION 1 (runs
      # repo-wide, same json.load parser, identical file:line error output).
      # A book-scoped Binder duplicate added nothing. The `check json` Binder
      # scope still exists for ad-hoc/CI use. Retired 2026-06-10.

      - id: mlsysim-check-registry-gates
        name: "MLSysIM: constants allowlist + no legacy imports + appendix verify"
        entry: ./book/binder check registry --scope tests
        language: system
        pass_filenames: false
        # The broad book/quarto/contents/.*\.qmd alternative already covers
        # appendix_assumptions.qmd, so no appendix-specific alternative is
        # listed (it would be dead). The gate runs on any chapter qmd because
        # appendix constants can be referenced from anywhere in the corpus.
        files: ^(mlsysim/mlsysim/core/constants\.py|mlsysim/mlsysim/core/defaults\.py|mlsysim/tests/test_constants_allowlist\.py|mlsysim/tests/test_no_legacy_constant_refs\.py|mlsysim/tests/test_appendix_constants\.py|book/tools/audit/generate_appendix_constants\.py|book/quarto/contents/.*\.qmd|labs/.*\.py|mlsysim/mlsysim/.*\.py)$

      - id: book-check-registry-sources
        name: "Book: check registry sources in LEGO cells"
        entry: ./book/binder check registry --scope sources
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-units
        name: "Book: check physics-engine unit conversion tests"
        # TODO: this is a test suite, not a static check. Consider moving
        # to CI; it currently runs in <5s but blocks commits on unrelated edits.
        entry: ./book/binder check units
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.qmd$

      - id: book-check-epub
        name: "Book: check EPUB source hygiene"
        # smoke + epubcheck scopes are default=False (require a built
        # EPUB / JRE). Run them in CI or manually with --all-scopes.
        # Triggers on source files the check inspects + the checker
        # implementation files.
        entry: ./book/binder check epub
        language: system
        pass_filenames: false
        files: ^book/quarto/contents/.*\.(svg|bib)$|^book/cli/commands/_epub_checks\.py$|^book/cli/commands/validate\.py$

  # ===========================================================================
  # SECTION 6: MANUAL — heavy / on-demand hooks (do NOT run on every commit)
  # Invoke explicitly:
  #   pre-commit run --hook-stage manual <hook-id>
  # ===========================================================================

  - repo: local
    hooks:
      - id: book-check-math-render-audit
        name: "Book: check math --scope render-audit (~10 min)"
        # Builds every chapter and scans rendered output for any LaTeX
        # that escaped MathJax. The empirical fallback for failure modes
        # the static math checks don't cover yet (and the source of truth
        # for the lightbox-tooltip leak class).
        entry: ./book/binder check math --scope render-audit
        language: system
        pass_filenames: false
        stages: [manual]

      - id: book-check-rendered-currency
        name: "Book: check rendered currency artifacts"
        # Scans cached HTML under book/quarto/_build/html-audit for visible
        # USD, doubled dollar signs, and raw LaTeX in prose. Populate that
        # cache first with book/tools/audit/fmt/render_html.sh.
        entry: ./book/binder check numbers --scope rendered-currency
        language: system
        pass_filenames: false
        stages: [manual]

      - id: book-cleanup-artifacts
        name: "Book: clean build artifacts"
        # Deletes _build/, .quarto/, index_files/. Running on every commit
        # destroys cached PDF builds (in particular, sequential vol1 → vol2
        # PDF builds fail with "FATAL Error opening book citations file"
        # afterward). Manual-only.
        entry: ./book/binder clean artifacts
        language: system
        pass_filenames: false
        stages: [manual]

  # ===========================================================================
  # SECTION 7: CI HYGIENE — GitHub Actions workflow safety
  # ===========================================================================

  - repo: local
    hooks:
      - id: ci-check-workflow-fork-safety
        name: "CI: Check workflows are safe for fork PRs"
        entry: python3 .github/scripts/check_workflow_fork_safety.py
        language: system
        pass_filenames: false
        files: ^\.github/workflows/.*\.(yml|yaml)$

  # ===========================================================================
  # SECTION 8: VAULT / STAFFML — vault-cli schema + ruff
  # ===========================================================================

  - repo: local
    hooks:
      - id: vault-schema-drift
        name: "Vault: Schema enums.py ↔ LinkML drift check"
        # Verifies schema/enums.py stays in sync with schema/question_schema.yaml
        # (the authoritative LinkML schema). Prevents the three-way drift that
        # caused the v0.1 migration defects. <1s.
        entry: python3 interviews/vault-cli/scripts/check_schema_sync.py
        language: system
        pass_filenames: false
        files: ^interviews/vault/schema/(enums\.py|question_schema\.yaml)$

  - repo: https://github.com/astral-sh/ruff-pre-commit
    # Mirrors `ruff check` in `🎯 StaffML · ✅ Validate (Vault)` so violations
    # are caught locally before they land on dev. Config lives in
    # interviews/vault-cli/pyproject.toml ([tool.ruff]).
    rev: v0.15.12
    hooks:
      - id: ruff
        name: "Vault: Lint vault-cli (ruff)"
        files: ^interviews/vault-cli/.*\.py$