|
| 1 | +"""Shared repo paths and file readers for the scripts under scripts/. |
| 2 | +
|
| 3 | +Twenty-odd scripts each spelled `Path(__file__).resolve().parent.parent`, and |
| 4 | +the same few readers — SKILL.md frontmatter, the fidelity fixtures, a JSON file |
| 5 | +that may be missing — were written out again wherever they were needed. This |
| 6 | +module is the one copy. It is stdlib-only at import time (pyyaml is imported |
| 7 | +only when frontmatter is parsed) and Python 3.9 compatible: the generator |
| 8 | +compatibility job compiles every script under 3.9. |
| 9 | +
|
| 10 | +Scripts import it as a sibling (`from _skill_io import ROOT`): running |
| 11 | +`python3 scripts/x.py` puts scripts/ on sys.path, and the pytest conftests do |
| 12 | +the same for in-process tests. |
| 13 | +
|
| 14 | +tools/ does not import it, deliberately. The create-master install copies |
| 15 | +tools/ without scripts/ (skill-catalog.json `bundle_paths`), so tools/ must |
| 16 | +stay self-contained; the dependency runs only scripts/ -> tools/. That is why |
| 17 | +tools/verify_sources.py and tools/master_builder.py keep their own |
| 18 | +`text.split("---", 2)` readers. |
| 19 | +
|
| 20 | +Two frontmatter readers, on purpose. They disagree on malformed input and each |
| 21 | +caller's behaviour is kept exactly (scripts/tests/test_skill_io.py pins both): |
| 22 | +
|
| 23 | +* `split_frontmatter` / `load_frontmatter` split on the first two `---` |
| 24 | + substrings anywhere in the text. Leading whitespace before the opening |
| 25 | + `---` is accepted; a `---` inside a value (`name: a---b`) ends the block |
| 26 | + early. Same semantics as tools/verify_sources.py's two readers. |
| 27 | +* `parse_frontmatter_lines` (validate.py) needs the opening and closing |
| 28 | + fences as whole lines, so `a---b` is read whole and a leading blank line |
| 29 | + means no frontmatter; a non-mapping or invalid block raises ValueError. |
| 30 | +
|
| 31 | +On every prebuilt SKILL.md the two return the same mapping. |
| 32 | +""" |
| 33 | +from __future__ import annotations |
| 34 | + |
| 35 | +import json |
| 36 | +import re |
| 37 | +from pathlib import Path |
| 38 | +from typing import Any, Dict, List, Optional, Tuple |
| 39 | + |
| 40 | +SCRIPTS_DIR = Path(__file__).resolve().parent |
| 41 | +ROOT = SCRIPTS_DIR.parent |
| 42 | +PREBUILT_DIR = ROOT / "prebuilt" |
| 43 | + |
| 44 | + |
| 45 | +# --- SKILL.md frontmatter ------------------------------------------------- |
| 46 | + |
| 47 | + |
| 48 | +def split_frontmatter(text: str) -> Optional[str]: |
| 49 | + """Raw YAML between the first two `---`, or None when there is no block. |
| 50 | +
|
| 51 | + No block means: fewer than two `---` in the text, or anything other than |
| 52 | + whitespace before the first one. |
| 53 | + """ |
| 54 | + parts = text.split("---", 2) |
| 55 | + if len(parts) < 3 or parts[0].strip(): |
| 56 | + return None |
| 57 | + return parts[1] |
| 58 | + |
| 59 | + |
| 60 | +def load_frontmatter(text: str) -> Any: |
| 61 | + """`split_frontmatter` parsed with yaml.safe_load; {} when absent or empty. |
| 62 | +
|
| 63 | + YAML errors propagate. A block that parses to a non-mapping is returned |
| 64 | + as parsed, not coerced — callers that call `.get` on it fail as before. |
| 65 | + """ |
| 66 | + raw = split_frontmatter(text) |
| 67 | + if raw is None: |
| 68 | + return {} |
| 69 | + import yaml |
| 70 | + |
| 71 | + return yaml.safe_load(raw) or {} |
| 72 | + |
| 73 | + |
| 74 | +def parse_frontmatter_lines(text: str, source: object = "<string>") -> Tuple[dict, str, List[str]]: |
| 75 | + """Line-fenced frontmatter: returns (mapping, body, lines). |
| 76 | +
|
| 77 | + The first line must be `---` (surrounding whitespace ignored) and the block |
| 78 | + ends at the next such line. Without both fences the result is |
| 79 | + ({}, text, lines). Invalid YAML or a non-mapping block raises ValueError |
| 80 | + prefixed with `source`. |
| 81 | + """ |
| 82 | + lines = text.splitlines() |
| 83 | + if not lines or lines[0].strip() != "---": |
| 84 | + return {}, text, lines |
| 85 | + |
| 86 | + end = None |
| 87 | + for i, line in enumerate(lines[1:], start=1): |
| 88 | + if line.strip() == "---": |
| 89 | + end = i |
| 90 | + break |
| 91 | + if end is None: |
| 92 | + return {}, text, lines |
| 93 | + |
| 94 | + import yaml |
| 95 | + |
| 96 | + try: |
| 97 | + fm = yaml.safe_load("\n".join(lines[1:end])) or {} |
| 98 | + except yaml.YAMLError as exc: |
| 99 | + raise ValueError(f"{source}: invalid YAML frontmatter — {exc}") from exc |
| 100 | + if not isinstance(fm, dict): |
| 101 | + raise ValueError(f"{source}: frontmatter is not a mapping") |
| 102 | + |
| 103 | + body = "\n".join(lines[end + 1 :]) |
| 104 | + return fm, body, lines |
| 105 | + |
| 106 | + |
| 107 | +# --- sources/*.md sections ------------------------------------------------ |
| 108 | + |
| 109 | + |
| 110 | +def parse_sections(text: str) -> List[Tuple[str, str]]: |
| 111 | + """按 ## 标题分段，返回 [(title, body), ...]""" |
| 112 | + sections = [] |
| 113 | + parts = re.split(r'^## ', text, flags=re.MULTILINE) |
| 114 | + for part in parts[1:]: |
| 115 | + lines = part.split('\n', 1) |
| 116 | + title = lines[0].strip() |
| 117 | + body = lines[1] if len(lines) > 1 else '' |
| 118 | + sections.append((title, body)) |
| 119 | + return sections |
| 120 | + |
| 121 | + |
| 122 | +# --- fixtures and JSON ---------------------------------------------------- |
| 123 | + |
| 124 | + |
| 125 | +def load_fixtures(prebuilt: Path = PREBUILT_DIR) -> Dict[str, List[dict]]: |
| 126 | + """{skill dir name: [fixture, ...]} from every `<prebuilt>/*/tests/fidelity.jsonl`. |
| 127 | +
|
| 128 | + Read with the platform default encoding, as both former copies did. |
| 129 | + """ |
| 130 | + fixtures: Dict[str, List[dict]] = {} |
| 131 | + for path in sorted(prebuilt.glob("*/tests/fidelity.jsonl")): |
| 132 | + fixtures[path.parent.parent.name] = [ |
| 133 | + json.loads(line) for line in path.read_text().splitlines() if line.strip() |
| 134 | + ] |
| 135 | + return fixtures |
| 136 | + |
| 137 | + |
| 138 | +def try_read_json(path: Path) -> Tuple[Any, Optional[Exception]]: |
| 139 | + """(parsed, None), or (None, error) when the file is unreadable or not JSON.""" |
| 140 | + try: |
| 141 | + return json.loads(path.read_text(encoding="utf-8")), None |
| 142 | + except (json.JSONDecodeError, OSError) as err: |
| 143 | + return None, err |
0 commit comments