|
| 1 | +"""Golden-record history site: the record of a CI golden change and the plots. |
| 2 | +
|
| 3 | +A golden-change PR leaves a record of its merge-queue run (collect); records |
| 4 | +of replayed commits come from golden_replay. render draws relative changes, |
| 5 | +paired run time and peak memory over the commit date and lists the events. |
| 6 | +""" |
| 7 | + |
| 8 | +import argparse |
| 9 | +import datetime |
| 10 | +import json |
| 11 | +import statistics |
| 12 | +import sys |
| 13 | +from pathlib import Path |
| 14 | + |
| 15 | +from .golden_changes import relative_changes |
| 16 | +from .golden_measure import measurements |
| 17 | +from .golden_replay import describe |
| 18 | + |
| 19 | + |
| 20 | +def collect(data, source, reference, runs, out): |
| 21 | + """Public record of a golden-record run whose solvers ran through wrap(). |
| 22 | +
|
| 23 | + The current runs log to runs/current.ranks, the runs of the merge base in |
| 24 | + the same job to runs/base.ranks, and the case directories in data hold |
| 25 | + the outputs of both. |
| 26 | + """ |
| 27 | + result = describe(source) |
| 28 | + result["reference"] = describe(reference)["neo2"] |
| 29 | + result["cases"] = {} |
| 30 | + result["changes"] = {"previous": {}, "main": {}} |
| 31 | + current = measurements(Path(runs) / "current.ranks") |
| 32 | + paired = measurements(Path(runs) / "base.ranks") |
| 33 | + for case, entry in sorted(current.items()): |
| 34 | + summary = Path(data) / case / "current" / "summary.h5" |
| 35 | + entry["status"] = "ok" if entry.pop("exit") == 0 and summary.exists() else "failed" |
| 36 | + if case in paired: |
| 37 | + entry["reference_wall_s"] = paired[case]["wall_s"] |
| 38 | + entry["reference_peak_rss_mib"] = paired[case]["peak_rss_mib"] |
| 39 | + base = Path(data) / case / "reference" / "summary.h5" |
| 40 | + if summary.exists() and base.exists(): |
| 41 | + # The reference is main just before the change, so it is the |
| 42 | + # previous event and main at once. |
| 43 | + changes = relative_changes(base, summary) |
| 44 | + result["changes"]["previous"][case] = result["changes"]["main"][case] = changes |
| 45 | + result["cases"][case] = entry |
| 46 | + Path(out).write_text(json.dumps(result, indent=1)) |
| 47 | + return result |
| 48 | + |
| 49 | + |
| 50 | +def render(records, out): |
| 51 | + """Write the public site: relative changes, run time and memory over time.""" |
| 52 | + import matplotlib |
| 53 | + matplotlib.use("Agg") |
| 54 | + import matplotlib.pyplot as plt |
| 55 | + |
| 56 | + out = Path(out) |
| 57 | + out.mkdir(parents=True, exist_ok=True) |
| 58 | + rows = sorted((json.loads(Path(p).read_text()) for p in records), key=lambda r: r["date"]) |
| 59 | + dates = {id(r): datetime.datetime.fromisoformat(r["date"]) for r in rows} |
| 60 | + cases = sorted({c for r in rows for c in r.get("cases", {})}) |
| 61 | + # Paired times: ratio to the reference run on the same node, times the |
| 62 | + # median reference time of the case, which removes drifts in node speed. |
| 63 | + ref = {c: statistics.median([r["cases"][c]["reference_wall_s"] for r in rows |
| 64 | + if c in r.get("cases", {}) and "reference_wall_s" in r["cases"][c]] |
| 65 | + or [1.0]) for c in cases} |
| 66 | + |
| 67 | + def change(case, entry, r, label): |
| 68 | + values = r.get("changes", {}).get(label, {}).get(case) |
| 69 | + numbers = [v for v in (values or {}).values() if v is not None] |
| 70 | + return max(max(numbers, default=0.0), 1e-17) if values else None |
| 71 | + |
| 72 | + def wall(case, entry, r): |
| 73 | + if entry.get("reference_wall_s"): |
| 74 | + return entry["wall_s"] / entry["reference_wall_s"] * ref[case] |
| 75 | + return entry["wall_s"] |
| 76 | + |
| 77 | + panels = [ |
| 78 | + ("changes.png", "Largest relative change against the previous event", "relative change", True, |
| 79 | + lambda c, e, r: change(c, e, r, "previous")), |
| 80 | + ("changes_main.png", "Largest relative change against current main", "relative change", True, |
| 81 | + lambda c, e, r: change(c, e, r, "main")), |
| 82 | + ("wall_time.png", "Wall time, paired against a reference on the same node", "wall time (s)", |
| 83 | + False, wall), |
| 84 | + ("memory.png", "Peak resident memory of the largest MPI rank", "peak memory (MiB)", False, |
| 85 | + lambda c, e, r: max(e["peak_rss_mib"], default=None)), |
| 86 | + ] |
| 87 | + for name, title, ylabel, log, value in panels: |
| 88 | + fig, ax = plt.subplots(figsize=(9, 4)) |
| 89 | + for case in cases: |
| 90 | + points = [(dates[id(r)], value(case, r["cases"][case], r), r) for r in rows |
| 91 | + if r.get("cases", {}).get(case, {}).get("status") == "ok"] |
| 92 | + points = [p for p in points if p[1] is not None] |
| 93 | + if not points: |
| 94 | + continue |
| 95 | + line, = ax.plot([p[0] for p in points], [p[1] for p in points], "-", lw=0.8, label=case) |
| 96 | + for when, y, r in points: |
| 97 | + source = r.get("event", {}).get("source") |
| 98 | + if source == "libneo": |
| 99 | + ax.plot(when, y, "^", color=line.get_color(), mfc="none") |
| 100 | + elif source: |
| 101 | + ax.plot(when, y, "o", color=line.get_color()) |
| 102 | + for r in rows: |
| 103 | + if r["build"] != "ok" or any(c.get("status") != "ok" for c in r["cases"].values()): |
| 104 | + ax.axvline(dates[id(r)], color="0.6", ls=":", lw=0.8) |
| 105 | + if log: |
| 106 | + ax.set_yscale("log") |
| 107 | + ax.set_title(title) |
| 108 | + ax.set_xlabel("commit date") |
| 109 | + ax.set_ylabel(ylabel) |
| 110 | + if ax.has_data(): |
| 111 | + ax.legend(fontsize="small", ncol=len(cases)) |
| 112 | + fig.autofmt_xdate() |
| 113 | + fig.tight_layout() |
| 114 | + fig.savefig(out / name, dpi=120) |
| 115 | + plt.close(fig) |
| 116 | + _write_html(rows, out) |
| 117 | + |
| 118 | + |
| 119 | +def _write_html(rows, out): |
| 120 | + import html |
| 121 | + repo = "https://github.com/itpplasma/NEO-2" |
| 122 | + body = [] |
| 123 | + for r in reversed(rows): |
| 124 | + event = r.get("event", {}) |
| 125 | + failed = [c for c, e in r["cases"].items() if e.get("status") != "ok"] |
| 126 | + status = "build failed" if r["build"] != "ok" else ( |
| 127 | + "failed: " + ", ".join(failed) if failed else "ok") |
| 128 | + pr = event.get("pr") |
| 129 | + issue = event.get("issue") |
| 130 | + cells = [r["date"][:10], html.escape(event.get("source", "")), |
| 131 | + f'<a href="{repo}/commit/{r["neo2"]}">{r["neo2"][:8]}</a>' |
| 132 | + + (f' (<a href="{repo}/pull/{pr}">#{pr}</a>)' if pr else ""), |
| 133 | + f'<a href="https://github.com/itpplasma/libneo/commit/{r["libneo"]}">{r["libneo"][:8]}</a>' |
| 134 | + if r.get("libneo") else "", |
| 135 | + html.escape(event.get("description", r.get("subject", ""))), |
| 136 | + html.escape(event.get("classification", "")) |
| 137 | + + (f' (<a href="{repo}/issues/{issue}">#{issue}</a>)' if issue else ""), |
| 138 | + status] |
| 139 | + body.append("<tr>" + "".join(f"<td>{c}</td>" for c in cells) + "</tr>") |
| 140 | + figures = "".join(f'<img src="{n}" alt="{n[:-4]}">' for n in |
| 141 | + ("changes.png", "changes_main.png", "wall_time.png", "memory.png")) |
| 142 | + (out / "index.html").write_text(f"""<!doctype html> |
| 143 | +<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"> |
| 144 | +<title>NEO-2 golden-record history</title> |
| 145 | +<style>body{{font-family:sans-serif;max-width:60rem;margin:auto;padding:0 1rem}} |
| 146 | +img{{max-width:100%}}table{{border-collapse:collapse;font-size:90%}} |
| 147 | +td,th{{border-bottom:1px solid #ccc;padding:.2rem .4rem;text-align:left;vertical-align:top}}</style> |
| 148 | +</head><body><h1>NEO-2 golden-record history</h1> |
| 149 | +<p>Each row is a build of NEO-2 with the libneo it used, run on the golden-record test cases. |
| 150 | +Relative change is max|x − x<sub>ref</sub>| / max|x<sub>ref</sub>| over each output dataset, |
| 151 | +largest over the datasets of a case. Filled circles mark NEO-2 events, open triangles libneo events, |
| 152 | +dotted lines builds or cases that failed.</p> |
| 153 | +{figures} |
| 154 | +<table><tr><th>date</th><th>event</th><th>NEO-2</th><th>libneo</th><th>change</th> |
| 155 | +<th>classification</th><th>status</th></tr> |
| 156 | +{''.join(body)} |
| 157 | +</table></body></html> |
| 158 | +""") |
| 159 | + |
| 160 | + |
| 161 | +def main(argv=None): |
| 162 | + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
| 163 | + sub = parser.add_subparsers(dest="command", required=True) |
| 164 | + p = sub.add_parser("collect", help="record of a golden-record CI run with measured solvers") |
| 165 | + p.add_argument("--data", required=True, help="golden_record directory the cases ran in") |
| 166 | + p.add_argument("--build", required=True, help="NEO-2 tree of the change") |
| 167 | + p.add_argument("--reference-build", required=True, help="NEO-2 tree of the merge base") |
| 168 | + p.add_argument("--runs", required=True, help="directory of current.ranks and base.ranks") |
| 169 | + p.add_argument("--out", required=True) |
| 170 | + p = sub.add_parser("render", help="write the site from record.json files") |
| 171 | + p.add_argument("records", nargs="+") |
| 172 | + p.add_argument("--out", required=True) |
| 173 | + a = parser.parse_args(argv) |
| 174 | + if a.command == "collect": |
| 175 | + collect(a.data, a.build, a.reference_build, a.runs, a.out) |
| 176 | + else: |
| 177 | + render(a.records, a.out) |
| 178 | + return 0 |
| 179 | + |
| 180 | + |
| 181 | +if __name__ == "__main__": |
| 182 | + sys.exit(main()) |
0 commit comments