diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..2abbdc4d --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,99 @@ +name: Benchmark + +permissions: + contents: read + +# All benchmark runs share one Moss project and the deterministic +# benchmark-ci- index, so concurrent jobs could race on first index +# creation and contaminate each other's latency numbers with cross-job +# load. Serialize globally; don't cancel a run that is already measuring. +concurrency: + group: moss-benchmark + cancel-in-progress: false + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + inputs: + update_baseline: + description: 'Update baseline.json with current results' + type: boolean + default: false + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # PyPI downloads occasionally break mid-stream on hosted runners + # (IncompleteRead / ProtocolError), and pip does not resume a + # partial wheel download — retry the whole install a few times. + for attempt in 1 2 3; do + if pip install -r benchmarks/ci/requirements.txt; then + exit 0 + fi + echo "pip install failed (attempt ${attempt}/3) — retrying in 15s" + sleep 15 + done + echo "pip install failed after 3 attempts" + exit 1 + + - name: Run benchmark suite + env: + MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} + MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} + # Fork PRs, and same-repo Dependabot PRs (which GitHub also denies + # normal secrets), cannot read repository secrets, so missing + # credentials are expected there and the suite may skip. On other + # trusted runs (push to main, same-repo human PRs, manual + # dispatch) missing secrets make the suite FAIL instead of + # passing as a green no-op. + ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]')) && '1' || '0' }} + run: | + # Baseline-update runs skip the regression comparison: comparing + # against the baseline being replaced would fail the run (and skip + # the copy step) exactly when an intentional change moved the numbers. + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && \ + [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json + else + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json \ + --latency-threshold=0.20 \ + --recall-threshold=0.05 + fi + + - name: Upload results artifact + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.sha }} + path: benchmark_results.json + retention-days: 90 + if: always() + + - name: Update baseline (manual trigger only) + if: >- + github.event_name == 'workflow_dispatch' && + github.event.inputs.update_baseline == 'true' + run: | + echo "Copying benchmark_results.json → benchmarks/ci/baseline.json" + cp benchmark_results.json benchmarks/ci/baseline.json + echo "New baseline (runner copy only — NOT committed):" + cat benchmarks/ci/baseline.json + echo "" + echo "To persist: download the benchmark-results-${{ github.sha }} artifact," + echo "copy it to benchmarks/ci/baseline.json, and commit." diff --git a/.gitignore b/.gitignore index b5028c58..9973c0b2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ node_modules/ dist/ .next/ coverage/ -.venv/ +.venv*/ env/ venv/ *.log @@ -27,3 +27,4 @@ skills-lock.json # Swift / SwiftPM build artifacts .build/ +benchmark_results.json diff --git a/AGENTS.md b/AGENTS.md index a4291351..fd69d9b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,3 +268,10 @@ The `.github/workflows/ci.yml` pipeline runs on push to `main` and on PRs: - `python-sdk-test` — matrix over Python 3.10–3.14 - `javascript-lint` — eslint - Separate release workflows publish to PyPI / npm on tagged releases + +The `.github/workflows/benchmark.yml` runs on push to `main` and on PRs: +- Runs the CI benchmark suite (`benchmarks/ci/`) measuring p50/p95/p99 latency and recall@k +- Compares against `benchmarks/ci/baseline.json` and fails on regressions exceeding thresholds +- Uploads `benchmark_results.json` as a workflow artifact for each run +- Supports `workflow_dispatch` with `update_baseline` input to refresh the baseline + diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore index 6dc235bd..156301f4 100644 --- a/benchmarks/.gitignore +++ b/benchmarks/.gitignore @@ -1,3 +1,5 @@ .env __pycache__/ embedding_server/__pycache__/ +ci/__pycache__/ +benchmark_results.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 378c3a99..b089ec16 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -77,4 +77,16 @@ EMBEDDING_DIMENSION=768 - **Queries**: 15 diverse search queries - **Warmup**: 3 rounds (excluded from measurements) - **Measured**: 50 rounds x 15 queries = 750 measurements per system -- **top_k**: 5 \ No newline at end of file +- **top_k**: 5 + +## CI Benchmark Suite + +For automated regression testing in CI, see [`benchmarks/ci/`](ci/). +The workflow runs on every push to `main` and on PRs; benchmark tests skip +when Moss credentials are unavailable (such as fork PRs). Authenticated runs +track p50/p95/p99 latency and recall@k per commit and compare against a +checked-in baseline, failing the build when regressions exceed the +configured thresholds. The latency guard activates once a baseline measured +on CI runners replaces the initial placeholder. + +See [`benchmarks/ci/README.md`](ci/README.md) for full documentation. \ No newline at end of file diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md new file mode 100644 index 00000000..8a355a7b --- /dev/null +++ b/benchmarks/ci/README.md @@ -0,0 +1,172 @@ +# CI Benchmark Suite — Latency & Recall Regression Guard + +Automated benchmark harness that runs on every push/PR to `main` and catches +performance regressions before they ship. + +## What it measures + +| Metric | Description | +|--------|-------------| +| **P50 / P95 / P99 latency** | End-to-end query latency (embedding + search) in ms | +| **Mean / Stdev** | Average and standard deviation of latency | +| **Recall@5** | Fraction of ground-truth top-5 docs returned in top 5 | +| **Recall@10** | Fraction of ground-truth top-10 docs returned in top 10 | + +All measurements use the Moss built-in embedding model (`moss-minilm`) with a +1,000-document subset of the benchmark corpus for CI speed. + +## Quick start + +### Prerequisites + +Set `MOSS_PROJECT_ID` and `MOSS_PROJECT_KEY` in your environment (or in a +`.env` file). + +### Run locally + +```bash +# Install dependencies +pip install -r benchmarks/ci/requirements.txt + +# Run the full suite +pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json + +# Skip regression checks (no baseline comparison) +pytest benchmarks/ci/ -v --benchmark-output=benchmark_results.json +``` + +### Regenerate ground truth + +Run this when the index data or model changes: + +```bash +python benchmarks/ci/generate_ground_truth.py +``` + +This queries Moss with `top_k=50` for each benchmark query and writes the +expected document IDs to `ground_truth.json`. Commit the updated file. +After a corpus or model change, pass `--recreate` so the index is rebuilt +from the current corpus before the ground truth is captured. + +`--recreate` deletion is guarded: only the derived `benchmark-ci-` +index is deleted without confirmation. If `MOSS_INDEX_NAME` overrides the +name, deletion additionally requires `--force`, and indexes outside the +`benchmark-ci-*` namespace are never deleted — so a stray environment +variable pointing at a shared or production index cannot be destroyed. + +> **Note**: the ground truth is a *ranking-stability reference* generated by +> Moss itself at a known-good commit — not an independent relevance judgment. +> The recall gate detects changes in retrieval behavior; an intentional +> relevance improvement will trip it and should be accompanied by a +> regenerated ground truth in the same PR. + +## How regression detection works + +The harness compares the current run's metrics against `baseline.json`: + +- **Latency**: Fails if P95 increases by more than the threshold (default 20%) +- **Recall**: Fails if Recall@5 **or** Recall@10 drops by more than the + threshold (default 5pp) + +The latency guard arms itself from the baseline file: + +- The checked-in placeholder declares `"latency_guard": "unarmed"` — the + latency test skips with a loud message while recall remains guarded, so + trusted runs are not red before the first baseline exists. +- To arm the guard, download the `benchmark-results-` artifact from a + trusted CI run and commit it as `baseline.json`. Artifacts carry no + `latency_guard` flag, so committing one arms the guard automatically. +- A zero p95 **without** the unarmed flag fails the run as a misconfigured + baseline — the guard can never be silently inactive by accident. + +Thresholds are configurable via CLI flags: + +```bash +# --latency-threshold: max fractional P95 increase (0.15 = 15%) +# --recall-threshold: max absolute recall@5 drop (0.03 = 3pp) +pytest benchmarks/ci/ -v \ + --latency-threshold=0.15 \ + --recall-threshold=0.03 +``` + +## Updating the baseline + +After a legitimate performance change (e.g., model upgrade, index config +change), update the baseline: + +1. Trigger the `Benchmark` workflow manually with `update_baseline=true`. + This runs the suite **without** the regression comparison (so an + intentional change can't fail its own baseline run) and produces a fresh + `benchmark-results-` artifact. It does **not** commit anything — + the runner copy is discarded when the job ends. +2. Download that artifact, copy it to `benchmarks/ci/baseline.json`, and + commit. Baselines should always come from CI runners — latency measured + on other hardware is not comparable. + +## Index naming and staleness protection + +The benchmark index name is derived from two content hashes — +`benchmark-ci--`: + +- **data signature** covers the model id, `DOC_COUNT`, and the exact corpus + slice being indexed; +- **build fingerprint** covers the installed SDK/bindings versions and the + Python SDK source tree, i.e. the code that builds the index. + +If any of those inputs change, the name changes and the index is rebuilt +from the current corpus by the current code — so a stale remote index can +never be silently benchmarked, and a PR that changes `create_index`, +document serialization, or the index/model build path always exercises that +path (an embedding change then surfaces as a recall regression instead of +passing against old embeddings). Set `MOSS_INDEX_NAME` to override the +derived name (this bypasses the staleness protection for the index itself; +the compatibility checks below still apply). + +Superseded indexes accumulate in the Moss project as signatures change; +clean them up occasionally with +`python benchmarks/ci/generate_ground_truth.py --prune` (only +`benchmark-ci-*` indexes are ever deleted). + +Three compatibility checks keep every comparison honest, and each **fails** +(never skips) on mismatch: + +- `ground_truth.json` embeds the corpus/model **signature** — recall is not + evaluated against ground truth generated from different inputs. +- `ground_truth.json` also embeds a **query-set hash** — adding, removing, + or editing `QUERIES` without regenerating trips it. +- `baseline.json`'s config (signature, query-set hash, doc count, rounds, + top_k) must match the current run's config — the regression guard refuses + to compare against a baseline captured under different benchmark inputs. + +Runs are serialized via a GitHub Actions `concurrency` group (without +cancelling in-progress runs): all jobs share one Moss project, so parallel +runs could race on index creation and contaminate each other's latency. + +## Credentials policy + +- **Trusted CI runs** (push to `main`, same-repo PRs, manual dispatch): + missing `MOSS_PROJECT_ID` / `MOSS_PROJECT_KEY` **fails** the suite — a + misconfigured secret must not turn the workflow into a green no-op. +- **Fork PRs**: secrets are not available to forks, so the workflow sets + `ALLOW_BENCHMARK_SKIP=1` and the suite skips cleanly. +- **Local runs** (no `CI` env var): missing credentials skip the suite. + +## CI integration + +The benchmark runs as a GitHub Actions job (`.github/workflows/benchmark.yml`). +Results are uploaded as artifacts named `benchmark-results-` and are +available for download from the Actions tab. + +## File overview + +| File | Purpose | +|------|---------| +| `test_bench_ci_moss.py` | Main test module (latency, recall, regression guard) | +| `bench_queries.py` | Shared query set + index config (used by tests and generator) | +| `conftest.py` | Pytest CLI flags | +| `generate_ground_truth.py` | One-time ground truth generator | +| `ground_truth.json` | Pre-computed expected results per query | +| `baseline.json` | Performance baseline for regression checks | +| `requirements.txt` | Python dependencies | diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json new file mode 100644 index 00000000..7e58e7da --- /dev/null +++ b/benchmarks/ci/baseline.json @@ -0,0 +1,28 @@ +{ + "commit": "f4732a1", + "timestamp": "2026-07-20T05:35:00+00:00", + "latency_guard": "unarmed", + "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are zero and 'latency_guard: unarmed' marks this file as the explicit placeholder: the latency regression test skips (loudly) instead of failing every trusted run. To arm the latency guard, run the Benchmark workflow on a trusted ref, download the benchmark-results- artifact, and commit it as this file \u2014 artifacts carry no latency_guard flag, so committing one arms the guard automatically. A zero p95 WITHOUT the unarmed flag fails the run as a misconfigured baseline. Latency baselines must come from CI runners; numbers from other hardware are not comparable.", + "latency_ms": { + "p50": 0, + "p95": 0, + "p99": 0, + "mean": 0, + "stdev": 0, + "count": 0 + }, + "recall": { + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "queries_evaluated": 15 + }, + "config": { + "doc_count": 1000, + "query_rounds": 20, + "warmup_rounds": 3, + "top_k_latency": 5, + "signature": "73d5e83176f7", + "query_set_hash": "70d932a5a939", + "query_count": 15 + } +} \ No newline at end of file diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py new file mode 100644 index 00000000..4909539a --- /dev/null +++ b/benchmarks/ci/bench_queries.py @@ -0,0 +1,117 @@ +"""Shared configuration for the CI benchmark suite. + +Both the benchmark tests (``test_bench_ci_moss.py``) and the ground-truth +generator (``generate_ground_truth.py``) import from this module so the +query set, index naming, and corpus signature can never drift between +generation and evaluation. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +INDEX_NAME_PREFIX = "benchmark-ci" +MODEL_ID = "moss-minilm" +DOC_COUNT = 1_000 + + +def load_corpus_slice(corpus_path: Path) -> list[dict[str, Any]]: + """Load the first ``DOC_COUNT`` documents of the shared corpus file.""" + with open(corpus_path) as f: + all_docs = json.load(f) + return all_docs[:DOC_COUNT] + + +def corpus_signature(docs: list[dict[str, Any]]) -> str: + """Short content hash of everything the benchmark index is built from. + + Covers the model id, DOC_COUNT, and the exact corpus slice that gets + indexed. Any change to those inputs yields a different signature — and + therefore a different index name via ``index_name_for`` — so a stale + remote index can never be silently reused against mismatched data. + """ + h = hashlib.sha256() + h.update(MODEL_ID.encode()) + h.update(str(DOC_COUNT).encode()) + for d in docs: + h.update(json.dumps(d, sort_keys=True).encode()) + return h.hexdigest()[:12] + + +def build_fingerprint() -> str: + """Fingerprint of every file belonging to the packages that build the index. + + Hashes the on-disk content of every file in the installed ``moss`` and + ``inferedge-moss-core`` distributions — Python sources, data files, and + native bindings (``.so``/``.pyd``) alike — rather than just their + version strings. A rebuilt native binding (e.g. a locally compiled wheel + during development, or a binding change that ships under an unchanged + version pin) still changes the file bytes even when the version string + doesn't, so it isn't missed. Any change to the indexing/build path + yields a new fingerprint — and therefore a new index name via + ``index_name_for`` — so the benchmark rebuilds the index and exercises + ``create_index``/document serialization instead of loading an index + built by older code (which could pass on stale embeddings). + """ + from importlib.metadata import PackageNotFoundError, distribution + + h = hashlib.sha256() + for pkg in ("moss", "inferedge-moss-core"): + try: + dist = distribution(pkg) + except PackageNotFoundError: + continue + h.update(f"{pkg}={dist.version}".encode()) + for rel_path in sorted(dist.files or (), key=str): + try: + data = rel_path.read_binary() + except (FileNotFoundError, IsADirectoryError, OSError): + continue + h.update(str(rel_path).encode()) + h.update(data) + return h.hexdigest()[:12] + + +def index_name_for(signature: str, fingerprint: str) -> str: + """Benchmark index name derived from data signature + build fingerprint. + + The name is the index's manifest: it changes whenever the corpus slice, + DOC_COUNT, model, or the SDK build path changes, so a stale remote index + can never be silently reused against mismatched inputs or code. + """ + return f"{INDEX_NAME_PREFIX}-{signature}-{fingerprint}" + + +def query_set_hash() -> str: + """Short hash of the benchmark query set. + + Stored in ``ground_truth.json`` and in every results file's config so + the recall test and the regression guard can detect a query set that + drifted from the one used at generation/baseline time. + """ + h = hashlib.sha256() + for q in QUERIES: + h.update(q.encode()) + h.update(b"\x00") + return h.hexdigest()[:12] + +QUERIES = [ + "neural network training data", + "anomaly detection patterns", + "computer vision image processing", + "natural language processing", + "reinforcement learning rewards", + "transfer learning pretrained models", + "distributed computing systems", + "cryptographic data encryption", + "database indexing performance", + "knowledge graph entities", + "generative adversarial networks", + "attention mechanism transformers", + "dimensionality reduction compression", + "federated learning privacy", + "stream processing pipelines", +] diff --git a/benchmarks/ci/conftest.py b/benchmarks/ci/conftest.py new file mode 100644 index 00000000..3c3fa05a --- /dev/null +++ b/benchmarks/ci/conftest.py @@ -0,0 +1,70 @@ +"""Pytest configuration for the CI benchmark suite. + +Adds custom CLI flags so the harness can be invoked as a standard pytest +run with configurable output paths and regression thresholds, and writes +the results artifact at session end. +""" + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("benchmark", "CI benchmark options") + group.addoption( + "--benchmark-output", + default="benchmark_results.json", + help="Path to write the JSON results file (default: benchmark_results.json)", + ) + group.addoption( + "--baseline-file", + default=None, + help="Path to baseline JSON for regression comparison. " + "If not provided, regression checks are skipped.", + ) + group.addoption( + "--latency-threshold", + type=float, + default=0.20, + help="Max allowed fractional increase in p95 latency vs baseline " + "(default: 0.20 = 20%%)", + ) + group.addoption( + "--recall-threshold", + type=float, + default=0.05, + help="Max allowed absolute decrease in recall@5 and recall@10 vs " + "baseline (default: 0.05 = 5 percentage points)", + ) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Serialize benchmark results after the run. + + Writing here (rather than in a test that must be collected last) means + the artifact is emitted regardless of test ordering, ``-x``/``--maxfail`` + early exits, or a failing regression guard — CI's ``if: always()`` + artifact upload always has real data to capture. + """ + config = session.config + if config.option.collectonly: + return + results = getattr(config, "_benchmark_results", None) + if results is None: + # No measurement fixture was ever instantiated (e.g. credentials + # missing and everything skipped). Still emit a stub so the artifact + # documents that nothing was measured instead of silently vanishing. + results = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "note": "no benchmark measurements ran in this session", + "latency_ms": {}, + "recall": {}, + } + output_path = Path(config.getoption("--benchmark-output")) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\nBenchmark results written to: {output_path}") diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py new file mode 100755 index 00000000..a9bfd902 --- /dev/null +++ b/benchmarks/ci/generate_ground_truth.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Generate ground truth for CI benchmark recall computation. + +Queries the Moss index with a large top_k and records the returned document +IDs as the "expected" set for each benchmark query. Run this once (or +whenever the index/model changes) and commit the output. + +.. note:: + This is a **ranking-stability reference**, not an independent relevance + judgment: the expected IDs come from Moss itself at a known-good commit. + The recall gate therefore detects *changes in retrieval behavior* (the + goal of a regression guard), and will also flag intentional relevance + improvements — regenerate and commit a new ground truth in that case. + +Usage:: + + # Ensure MOSS_PROJECT_ID and MOSS_PROJECT_KEY are set + python benchmarks/ci/generate_ground_truth.py + + # After a corpus or model change, rebuild the index first: + python benchmarks/ci/generate_ground_truth.py --recreate + +Output is written to ``benchmarks/ci/ground_truth.json``. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path + +from bench_queries import ( + DOC_COUNT, + INDEX_NAME_PREFIX, + MODEL_ID, + QUERIES, + build_fingerprint, + corpus_signature, + index_name_for, + load_corpus_slice, + query_set_hash, +) +from dotenv import load_dotenv + +load_dotenv() + +# Fetch a generous top_k so recall@5 and recall@10 can be evaluated +# against a superset of relevant results. +GROUND_TRUTH_TOP_K = 50 + + +def _corpus_slice() -> list[dict]: + corpus_path = Path(__file__).resolve().parent.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + print(f"Error: Corpus file not found: {corpus_path}") + sys.exit(1) + return load_corpus_slice(corpus_path) + + +async def _create_index(client, index_name: str, corpus_slice: list[dict]) -> None: + from moss import DocumentInfo + + docs = [ + DocumentInfo(id=d["id"], text=d["text"], metadata=d.get("metadata")) + for d in corpus_slice + ] + result = await client.create_index(index_name, docs, MODEL_ID) + print(f"Created index '{index_name}' with {result.doc_count} docs") + + +def _guard_deletion(index_name: str, derived_name: str, force: bool) -> None: + """Refuse to delete indexes that are not clearly benchmark-owned. + + MOSS_INDEX_NAME is a documented override, so a developer whose + environment points at a shared or production Moss project could + otherwise aim --recreate at a non-benchmark index and destroy it. + """ + if index_name == derived_name: + return # the derived benchmark index — always safe to recreate + if not index_name.startswith(f"{INDEX_NAME_PREFIX}-"): + print( + f"Error: refusing to delete index '{index_name}' — it is outside " + f"the benchmark namespace ('{INDEX_NAME_PREFIX}-*'). Unset " + "MOSS_INDEX_NAME or point it at a benchmark index." + ) + sys.exit(1) + if not force: + print( + f"Error: MOSS_INDEX_NAME overrides the derived name " + f"('{index_name}' != '{derived_name}'). Pass --force to confirm " + "deleting the overridden benchmark index." + ) + sys.exit(1) + + +async def main(recreate: bool, force: bool, prune: bool) -> dict: + from moss import MossClient, QueryOptions + + project_id = os.getenv("MOSS_PROJECT_ID") + project_key = os.getenv("MOSS_PROJECT_KEY") + # Same derivation as the benchmark tests: the index name embeds the + # corpus/model signature and the SDK build fingerprint, so generation + # and evaluation can never target indexes built from different inputs + # or by different indexing code. + corpus_slice = _corpus_slice() + signature = corpus_signature(corpus_slice) + derived_name = index_name_for(signature, build_fingerprint()) + index_name = os.getenv("MOSS_INDEX_NAME") or derived_name + + if not project_id or not project_key: + print("Error: MOSS_PROJECT_ID and MOSS_PROJECT_KEY must be set.") + sys.exit(1) + + client = MossClient(project_id, project_key) + + # Determine existence explicitly (rather than treating any get_index + # failure as "missing") so auth/network errors surface instead of + # silently triggering index creation. + existing = {idx.name for idx in await client.list_indexes()} + + if index_name in existing and recreate: + _guard_deletion(index_name, derived_name, force) + print(f"--recreate: deleting existing index '{index_name}'") + await client.delete_index(index_name) + existing.discard(index_name) + + if index_name in existing: + print(f"Using existing index '{index_name}'") + else: + await _create_index(client, index_name, corpus_slice) + + await client.load_index(index_name) + + # Query each benchmark query with a large top_k. + ground_truth: dict[str, list[str]] = {} + for q in QUERIES: + result = await client.query( + index_name, + q, + QueryOptions(top_k=GROUND_TRUTH_TOP_K, alpha=1), + ) + doc_ids = [doc.id for doc in result.docs] + ground_truth[q] = doc_ids + print(f" '{q}' → {len(doc_ids)} results") + + output = { + "model": MODEL_ID, + "top_k": GROUND_TRUTH_TOP_K, + "index_name": index_name, + "doc_count": DOC_COUNT, + # Validated by the benchmark tests: recall is only evaluated when the + # current corpus/model signature AND query set match the ones used + # at generation. + "signature": signature, + "query_set": {"hash": query_set_hash(), "count": len(QUERIES)}, + "queries": ground_truth, + } + + if prune: + # Old corpus/SDK revisions leave benchmark-ci-* indexes behind; + # remove everything in the benchmark namespace except the index we + # just used. Never touches indexes outside the namespace. + stale = sorted( + n + for n in existing + if n != index_name + and (n == INDEX_NAME_PREFIX or n.startswith(f"{INDEX_NAME_PREFIX}-")) + ) + for n in stale: + print(f"--prune: deleting stale benchmark index '{n}'") + await client.delete_index(n) + if not stale: + print("--prune: no stale benchmark indexes found") + + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--recreate", + action="store_true", + help="Delete and rebuild the benchmark index from the corpus before " + "querying. Required after a corpus or embedding-model change so the " + "ground truth reflects the current data.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Confirm --recreate deletion when MOSS_INDEX_NAME overrides the " + "derived index name. Only benchmark-namespace indexes " + "(benchmark-ci-*) can be deleted even with this flag.", + ) + parser.add_argument( + "--prune", + action="store_true", + help="After generating, delete leftover benchmark-namespace indexes " + "(benchmark-ci-*) from earlier corpus/SDK revisions. Indexes outside " + "the benchmark namespace are never touched.", + ) + args = parser.parse_args() + output = asyncio.run(main(recreate=args.recreate, force=args.force, prune=args.prune)) + + output_path = Path(__file__).resolve().parent / "ground_truth.json" + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + + print(f"\nGround truth written to: {output_path}") + print(f"Queries: {len(output['queries'])}") diff --git a/benchmarks/ci/ground_truth.json b/benchmarks/ci/ground_truth.json new file mode 100644 index 00000000..0216e608 --- /dev/null +++ b/benchmarks/ci/ground_truth.json @@ -0,0 +1,793 @@ +{ + "model": "moss-minilm", + "top_k": 50, + "index_name": "benchmark-ci-73d5e83176f7", + "doc_count": 1000, + "signature": "73d5e83176f7", + "query_set": { + "hash": "70d932a5a939", + "count": 15 + }, + "queries": { + "neural network training data": [ + "doc_113", + "doc_877", + "doc_91", + "doc_984", + "doc_200", + "doc_160", + "doc_946", + "doc_593", + "doc_349", + "doc_536", + "doc_325", + "doc_763", + "doc_23", + "doc_297", + "doc_460", + "doc_601", + "doc_865", + "doc_965", + "doc_62", + "doc_698", + "doc_128", + "doc_715", + "doc_455", + "doc_992", + "doc_229", + "doc_215", + "doc_447", + "doc_578", + "doc_243", + "doc_360", + "doc_510", + "doc_675", + "doc_157", + "doc_602", + "doc_794", + "doc_202", + "doc_824", + "doc_493", + "doc_503", + "doc_210", + "doc_641", + "doc_872", + "doc_633", + "doc_472", + "doc_6", + "doc_803", + "doc_329", + "doc_971", + "doc_350", + "doc_500" + ], + "anomaly detection patterns": [ + "doc_267", + "doc_249", + "doc_345", + "doc_700", + "doc_289", + "doc_806", + "doc_154", + "doc_929", + "doc_837", + "doc_612", + "doc_373", + "doc_120", + "doc_242", + "doc_896", + "doc_853", + "doc_495", + "doc_397", + "doc_854", + "doc_873", + "doc_861", + "doc_563", + "doc_170", + "doc_487", + "doc_855", + "doc_766", + "doc_13", + "doc_377", + "doc_758", + "doc_827", + "doc_735", + "doc_5", + "doc_57", + "doc_423", + "doc_381", + "doc_871", + "doc_471", + "doc_109", + "doc_180", + "doc_783", + "doc_643", + "doc_327", + "doc_459", + "doc_211", + "doc_507", + "doc_682", + "doc_92", + "doc_438", + "doc_58", + "doc_828", + "doc_29" + ], + "computer vision image processing": [ + "doc_165", + "doc_847", + "doc_878", + "doc_388", + "doc_843", + "doc_597", + "doc_80", + "doc_89", + "doc_71", + "doc_218", + "doc_19", + "doc_257", + "doc_129", + "doc_0", + "doc_677", + "doc_755", + "doc_124", + "doc_717", + "doc_548", + "doc_748", + "doc_805", + "doc_489", + "doc_366", + "doc_228", + "doc_830", + "doc_598", + "doc_520", + "doc_769", + "doc_815", + "doc_191", + "doc_684", + "doc_709", + "doc_193", + "doc_371", + "doc_547", + "doc_626", + "doc_690", + "doc_106", + "doc_294", + "doc_640", + "doc_3", + "doc_913", + "doc_33", + "doc_661", + "doc_248", + "doc_760", + "doc_721", + "doc_741", + "doc_139", + "doc_729" + ], + "natural language processing": [ + "doc_114", + "doc_278", + "doc_317", + "doc_181", + "doc_968", + "doc_943", + "doc_161", + "doc_112", + "doc_355", + "doc_32", + "doc_281", + "doc_7", + "doc_208", + "doc_30", + "doc_733", + "doc_178", + "doc_958", + "doc_988", + "doc_797", + "doc_978", + "doc_141", + "doc_670", + "doc_912", + "doc_594", + "doc_619", + "doc_867", + "doc_163", + "doc_879", + "doc_998", + "doc_826", + "doc_359", + "doc_589", + "doc_162", + "doc_842", + "doc_506", + "doc_546", + "doc_851", + "doc_93", + "doc_457", + "doc_502", + "doc_753", + "doc_901", + "doc_718", + "doc_26", + "doc_236", + "doc_754", + "doc_39", + "doc_949", + "doc_596", + "doc_868" + ], + "reinforcement learning rewards": [ + "doc_175", + "doc_318", + "doc_292", + "doc_813", + "doc_960", + "doc_846", + "doc_751", + "doc_60", + "doc_209", + "doc_955", + "doc_973", + "doc_911", + "doc_96", + "doc_707", + "doc_585", + "doc_897", + "doc_461", + "doc_577", + "doc_639", + "doc_644", + "doc_221", + "doc_655", + "doc_391", + "doc_818", + "doc_213", + "doc_850", + "doc_785", + "doc_917", + "doc_953", + "doc_320", + "doc_263", + "doc_683", + "doc_538", + "doc_942", + "doc_63", + "doc_449", + "doc_674", + "doc_557", + "doc_549", + "doc_492", + "doc_212", + "doc_342", + "doc_491", + "doc_584", + "doc_991", + "doc_386", + "doc_857", + "doc_713", + "doc_50", + "doc_952" + ], + "transfer learning pretrained models": [ + "doc_273", + "doc_339", + "doc_857", + "doc_108", + "doc_841", + "doc_50", + "doc_203", + "doc_177", + "doc_889", + "doc_246", + "doc_45", + "doc_386", + "doc_398", + "doc_991", + "doc_704", + "doc_527", + "doc_952", + "doc_909", + "doc_903", + "doc_197", + "doc_316", + "doc_587", + "doc_951", + "doc_895", + "doc_916", + "doc_519", + "doc_840", + "doc_713", + "doc_454", + "doc_521", + "doc_173", + "doc_115", + "doc_736", + "doc_255", + "doc_156", + "doc_408", + "doc_352", + "doc_580", + "doc_131", + "doc_574", + "doc_653", + "doc_504", + "doc_526", + "doc_405", + "doc_559", + "doc_323", + "doc_47", + "doc_74", + "doc_792", + "doc_225" + ], + "distributed computing systems": [ + "doc_479", + "doc_306", + "doc_940", + "doc_933", + "doc_997", + "doc_514", + "doc_511", + "doc_98", + "doc_849", + "doc_607", + "doc_85", + "doc_714", + "doc_369", + "doc_224", + "doc_277", + "doc_76", + "doc_994", + "doc_79", + "doc_599", + "doc_341", + "doc_146", + "doc_266", + "doc_782", + "doc_40", + "doc_908", + "doc_779", + "doc_227", + "doc_222", + "doc_543", + "doc_832", + "doc_64", + "doc_392", + "doc_919", + "doc_795", + "doc_443", + "doc_880", + "doc_15", + "doc_790", + "doc_608", + "doc_435", + "doc_615", + "doc_706", + "doc_614", + "doc_250", + "doc_192", + "doc_650", + "doc_446", + "doc_368", + "doc_560", + "doc_750" + ], + "cryptographic data encryption": [ + "doc_364", + "doc_336", + "doc_2", + "doc_486", + "doc_477", + "doc_298", + "doc_46", + "doc_312", + "doc_233", + "doc_107", + "doc_335", + "doc_31", + "doc_164", + "doc_689", + "doc_836", + "doc_666", + "doc_54", + "doc_712", + "doc_442", + "doc_490", + "doc_230", + "doc_286", + "doc_989", + "doc_483", + "doc_185", + "doc_8", + "doc_976", + "doc_948", + "doc_571", + "doc_656", + "doc_132", + "doc_282", + "doc_12", + "doc_110", + "doc_309", + "doc_950", + "doc_605", + "doc_699", + "doc_972", + "doc_463", + "doc_635", + "doc_378", + "doc_111", + "doc_431", + "doc_413", + "doc_923", + "doc_217", + "doc_881", + "doc_41", + "doc_226" + ], + "database indexing performance": [ + "doc_83", + "doc_475", + "doc_910", + "doc_253", + "doc_182", + "doc_647", + "doc_474", + "doc_637", + "doc_37", + "doc_696", + "doc_367", + "doc_102", + "doc_570", + "doc_152", + "doc_703", + "doc_321", + "doc_874", + "doc_133", + "doc_284", + "doc_441", + "doc_351", + "doc_436", + "doc_234", + "doc_702", + "doc_488", + "doc_767", + "doc_412", + "doc_362", + "doc_275", + "doc_216", + "doc_532", + "doc_432", + "doc_376", + "doc_928", + "doc_18", + "doc_49", + "doc_153", + "doc_694", + "doc_123", + "doc_743", + "doc_802", + "doc_95", + "doc_415", + "doc_970", + "doc_544", + "doc_926", + "doc_150", + "doc_937", + "doc_648", + "doc_44" + ], + "knowledge graph entities": [ + "doc_370", + "doc_810", + "doc_168", + "doc_609", + "doc_287", + "doc_610", + "doc_247", + "doc_894", + "doc_196", + "doc_380", + "doc_481", + "doc_875", + "doc_272", + "doc_957", + "doc_869", + "doc_775", + "doc_777", + "doc_900", + "doc_723", + "doc_402", + "doc_967", + "doc_9", + "doc_793", + "doc_244", + "doc_220", + "doc_28", + "doc_685", + "doc_409", + "doc_669", + "doc_884", + "doc_800", + "doc_975", + "doc_737", + "doc_924", + "doc_934", + "doc_705", + "doc_856", + "doc_465", + "doc_167", + "doc_796", + "doc_961", + "doc_764", + "doc_542", + "doc_556", + "doc_920", + "doc_87", + "doc_888", + "doc_280", + "doc_732", + "doc_466" + ], + "generative adversarial networks": [ + "doc_265", + "doc_350", + "doc_20", + "doc_67", + "doc_985", + "doc_340", + "doc_268", + "doc_659", + "doc_918", + "doc_625", + "doc_784", + "doc_145", + "doc_421", + "doc_290", + "doc_664", + "doc_899", + "doc_892", + "doc_291", + "doc_261", + "doc_130", + "doc_983", + "doc_528", + "doc_319", + "doc_823", + "doc_679", + "doc_588", + "doc_839", + "doc_931", + "doc_329", + "doc_344", + "doc_870", + "doc_332", + "doc_627", + "doc_353", + "doc_631", + "doc_375", + "doc_657", + "doc_523", + "doc_499", + "doc_788", + "doc_572", + "doc_693", + "doc_576", + "doc_403", + "doc_555", + "doc_890", + "doc_256", + "doc_840", + "doc_108", + "doc_713" + ], + "attention mechanism transformers": [ + "doc_428", + "doc_756", + "doc_384", + "doc_906", + "doc_21", + "doc_358", + "doc_907", + "doc_35", + "doc_845", + "doc_822", + "doc_382", + "doc_59", + "doc_692", + "doc_862", + "doc_687", + "doc_620", + "doc_658", + "doc_834", + "doc_866", + "doc_636", + "doc_36", + "doc_804", + "doc_389", + "doc_691", + "doc_104", + "doc_237", + "doc_680", + "doc_765", + "doc_407", + "doc_84", + "doc_184", + "doc_979", + "doc_662", + "doc_761", + "doc_628", + "doc_885", + "doc_665", + "doc_781", + "doc_245", + "doc_603", + "doc_533", + "doc_177", + "doc_890", + "doc_385", + "doc_916", + "doc_952", + "doc_580", + "doc_895", + "doc_197", + "doc_108" + ], + "dimensionality reduction compression": [ + "doc_140", + "doc_95", + "doc_119", + "doc_544", + "doc_343", + "doc_357", + "doc_554", + "doc_445", + "doc_395", + "doc_937", + "doc_334", + "doc_970", + "doc_678", + "doc_11", + "doc_101", + "doc_155", + "doc_618", + "doc_4", + "doc_10", + "doc_150", + "doc_926", + "doc_44", + "doc_194", + "doc_982", + "doc_333", + "doc_757", + "doc_668", + "doc_305", + "doc_478", + "doc_427", + "doc_55", + "doc_914", + "doc_524", + "doc_464", + "doc_902", + "doc_776", + "doc_190", + "doc_759", + "doc_993", + "doc_774", + "doc_72", + "doc_770", + "doc_819", + "doc_821", + "doc_219", + "doc_681", + "doc_590", + "doc_434", + "doc_648", + "doc_568" + ], + "federated learning privacy": [ + "doc_887", + "doc_337", + "doc_829", + "doc_347", + "doc_966", + "doc_716", + "doc_383", + "doc_977", + "doc_728", + "doc_835", + "doc_16", + "doc_354", + "doc_264", + "doc_251", + "doc_426", + "doc_518", + "doc_799", + "doc_962", + "doc_126", + "doc_143", + "doc_974", + "doc_322", + "doc_260", + "doc_673", + "doc_125", + "doc_254", + "doc_922", + "doc_905", + "doc_747", + "doc_201", + "doc_749", + "doc_515", + "doc_223", + "doc_604", + "doc_414", + "doc_817", + "doc_135", + "doc_745", + "doc_387", + "doc_169", + "doc_206", + "doc_65", + "doc_801", + "doc_271", + "doc_762", + "doc_573", + "doc_326", + "doc_622", + "doc_606", + "doc_531" + ], + "stream processing pipelines": [ + "doc_560", + "doc_505", + "doc_315", + "doc_250", + "doc_410", + "doc_663", + "doc_204", + "doc_848", + "doc_446", + "doc_708", + "doc_935", + "doc_558", + "doc_43", + "doc_750", + "doc_615", + "doc_509", + "doc_48", + "doc_53", + "doc_893", + "doc_308", + "doc_614", + "doc_959", + "doc_706", + "doc_241", + "doc_551", + "doc_686", + "doc_1", + "doc_195", + "doc_650", + "doc_778", + "doc_17", + "doc_368", + "doc_734", + "doc_986", + "doc_117", + "doc_38", + "doc_651", + "doc_148", + "doc_310", + "doc_73", + "doc_34", + "doc_56", + "doc_638", + "doc_174", + "doc_172", + "doc_623", + "doc_78", + "doc_183", + "doc_772", + "doc_660" + ] + } +} \ No newline at end of file diff --git a/benchmarks/ci/requirements.txt b/benchmarks/ci/requirements.txt new file mode 100644 index 00000000..60474cae --- /dev/null +++ b/benchmarks/ci/requirements.txt @@ -0,0 +1,8 @@ +# CI benchmark harness — minimal dependencies +# NOTE: install from the repo root (pip resolves this path against the CWD): +# pip install -r benchmarks/ci/requirements.txt +# Versions are pinned exactly so benchmark numbers stay attributable to +# repository changes rather than dependency drift; bump deliberately. +./sdks/python/sdk +pytest==9.1.1 +python-dotenv==1.2.2 diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py new file mode 100644 index 00000000..dde64557 --- /dev/null +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -0,0 +1,542 @@ +"""CI Benchmark Suite — Latency and Recall for Moss. + +Runs a fixed query set against a Moss index and records: + - p50 / p95 / p99 / mean latency (ms) + - recall@5 and recall@10 vs pre-computed ground truth + +Results are written to a JSON file (``--benchmark-output``) and optionally +compared against a checked-in baseline (``--baseline-file``) to catch +performance regressions. + +Usage:: + + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json +""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +import statistics +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from bench_queries import ( + DOC_COUNT, + MODEL_ID, + QUERIES, + build_fingerprint, + corpus_signature, + index_name_for, + load_corpus_slice, + query_set_hash, +) +try: + from dotenv import load_dotenv +except ModuleNotFoundError: # pragma: no cover - optional outside benchmarks/ci + + def load_dotenv() -> None: + return None + + +load_dotenv() + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +CI_DIR = Path(__file__).resolve().parent + +# DOC_COUNT (1K subset of the full 100K corpus, for CI speed), the query set, +# and the model id are shared with generate_ground_truth.py via bench_queries. +TOP_K_LATENCY = 5 +TOP_K_RECALL_5 = 5 +TOP_K_RECALL_10 = 10 +WARMUP_ROUNDS = 3 +QUERY_ROUNDS = 20 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _percentile(values: list[float], p: float) -> float: + """Compute the *p*-th percentile from a **sorted** list of values.""" + if not values: + return 0.0 + idx = max(math.ceil(p * len(values)) - 1, 0) + return values[idx] + + +_loop: asyncio.AbstractEventLoop | None = None + + +def _run(coro): + """Run *coro* on a single shared event loop. + + ``asyncio.get_event_loop()`` is deprecated (and raises on Python 3.14) + when no loop is running; ``asyncio.run()`` would create a fresh loop per + call, breaking clients that bind connections to the first loop. A single + explicit loop shared across the session avoids both problems. + """ + global _loop + if _loop is None: + _loop = asyncio.new_event_loop() + asyncio.set_event_loop(_loop) + return _loop.run_until_complete(coro) + + +def _git_sha() -> str: + """Return the short git SHA of HEAD, or 'unknown'.""" + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except (subprocess.SubprocessError, OSError): + return "unknown" + + +def _missing_required_input(message: str): + """Handle a missing benchmark prerequisite (inputs, measurement data, …). + + Skip on fork PRs (``ALLOW_BENCHMARK_SKIP=1``) and local runs, but FAIL + in trusted CI — a missing prerequisite must not turn the benchmark + workflow into a green no-op. + """ + if os.getenv("ALLOW_BENCHMARK_SKIP") == "1": + pytest.skip(f"{message} — fork PR, skipping benchmarks") + if os.getenv("CI"): + pytest.fail(f"{message} — must not silently pass in a trusted CI run") + pytest.skip(message) + + +# Config keys that must match between the current run and the baseline for a +# regression comparison to be meaningful. index_name is excluded: it may be +# overridden via MOSS_INDEX_NAME without changing what is measured. +BASELINE_COMPAT_KEYS = ( + "signature", + "query_set_hash", + "doc_count", + "query_rounds", + "warmup_rounds", + "top_k_latency", +) + + +def _assert_baseline_compatible(baseline: dict, benchmark_results: dict) -> None: + """Fail when the baseline was captured under different benchmark inputs. + + Comparing against a baseline built from another corpus/model, query set, + or measurement config silently masks (or fabricates) regressions. + """ + current = {k: benchmark_results.get("config", {}).get(k) for k in BASELINE_COMPAT_KEYS} + base = {k: baseline.get("config", {}).get(k) for k in BASELINE_COMPAT_KEYS} + if base != current: + diffs = {k: (base[k], current[k]) for k in BASELINE_COMPAT_KEYS if base[k] != current[k]} + pytest.fail( + "baseline.json is not comparable to this run — config mismatch " + f"(baseline vs current): {diffs}. Regenerate the baseline via the " + "Benchmark workflow with update_baseline=true and commit the artifact." + ) + + +# --------------------------------------------------------------------------- +# Session-scoped fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def corpus_slice() -> list[dict]: + """The exact corpus slice the benchmark index is built from.""" + corpus_path = CI_DIR.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + _missing_required_input(f"Corpus file not found: {corpus_path}") + return load_corpus_slice(corpus_path) + + +@pytest.fixture(scope="session") +def corpus_sig(corpus_slice) -> str: + """Content signature of model + DOC_COUNT + corpus slice.""" + return corpus_signature(corpus_slice) + + +@pytest.fixture(scope="session") +def build_fp() -> str: + """Fingerprint of the index build path (SDK versions + source tree).""" + return build_fingerprint() + + +@pytest.fixture(scope="session") +def moss_client(corpus_slice, corpus_sig, build_fp): + """Create a MossClient and load the benchmark index once per session. + + The index name embeds ``corpus_sig`` (corpus/DOC_COUNT/model) AND + ``build_fp`` (SDK versions + source), so an index built from different + data or by different indexing code can never be silently reused — + mismatched inputs produce a different name and the index is (re)created + from the current corpus by the current code, exercising the full + create_index/document-serialization path whenever it changes. + """ + project_id = os.getenv("MOSS_PROJECT_ID") + project_key = os.getenv("MOSS_PROJECT_KEY") + if not project_id or not project_key: + if os.getenv("ALLOW_BENCHMARK_SKIP") == "1": + pytest.skip( + "MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — fork PR without " + "secrets, skipping benchmarks" + ) + if os.getenv("CI"): + pytest.fail( + "MOSS_PROJECT_ID / MOSS_PROJECT_KEY are not set in a trusted CI " + "run — the benchmark workflow would otherwise pass as a green " + "no-op. Configure the repository secrets, or export " + "ALLOW_BENCHMARK_SKIP=1 for runs that legitimately lack them." + ) + pytest.skip("MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — skipping benchmarks") + + # Import lazily (and only once credentials are known to exist) — Moss + # native bindings may not be installed in every env. + from moss import DocumentInfo, MossClient + + index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig, build_fp) + + async def _setup(): + # Construct the client inside the coroutine, after _run() has + # created and installed the shared event loop — MossClient may bind + # an async session or call get_event_loop() internally, and must do + # so against the loop it will actually run on. + client = MossClient(project_id, project_key) + + # Determine existence explicitly (rather than treating any get_index + # failure as "missing") so auth/network errors surface instead of + # silently triggering index creation. + existing = {idx.name for idx in await client.list_indexes()} + if index_name not in existing: + docs = [ + DocumentInfo( + id=d["id"], + text=d["text"], + metadata=d.get("metadata"), + ) + for d in corpus_slice + ] + await client.create_index(index_name, docs, MODEL_ID) + + await client.load_index(index_name) + return client, index_name + + client, index_name = _run(_setup()) + yield client, index_name + + +@pytest.fixture(scope="session") +def ground_truth(corpus_sig) -> dict[str, list[str]]: + """Load pre-computed ground truth document IDs per query. + + Fails (not skips) when the ground truth was generated from a different + corpus/model/DOC_COUNT — evaluating recall against mismatched expected + ids would mask corpus or model regressions. + """ + gt_path = CI_DIR / "ground_truth.json" + if not gt_path.exists(): + _missing_required_input(f"Ground truth file not found: {gt_path}") + with open(gt_path) as f: + data = json.load(f) + gt_sig = data.get("signature") + if gt_sig != corpus_sig: + pytest.fail( + f"ground_truth.json signature {gt_sig!r} does not match the current " + f"corpus/model signature {corpus_sig!r} — the corpus, DOC_COUNT, or " + "model changed since generation. Regenerate with: " + "python benchmarks/ci/generate_ground_truth.py --recreate" + ) + gt_query_hash = data.get("query_set", {}).get("hash") + if gt_query_hash != query_set_hash(): + pytest.fail( + f"ground_truth.json query-set hash {gt_query_hash!r} does not match " + f"the current query set ({query_set_hash()!r}) — QUERIES changed " + "since generation. Regenerate with: " + "python benchmarks/ci/generate_ground_truth.py" + ) + return data.get("queries", {}) + + +@pytest.fixture(scope="session") +def benchmark_results(request, corpus_sig, build_fp) -> dict: + """Mutable dict that accumulates results across tests in this session. + + Registered on the pytest config so ``pytest_sessionfinish`` (in + conftest.py) serializes it to JSON after the run — regardless of test + ordering, ``-x``/``--maxfail``, or guard failures. + """ + results = { + "commit": _git_sha(), + "timestamp": datetime.now(timezone.utc).isoformat(), + "config": { + "doc_count": DOC_COUNT, + "query_rounds": QUERY_ROUNDS, + "warmup_rounds": WARMUP_ROUNDS, + "top_k_latency": TOP_K_LATENCY, + "signature": corpus_sig, + "query_set_hash": query_set_hash(), + "query_count": len(QUERIES), + "build_fingerprint": build_fp, + "index_name": os.getenv("MOSS_INDEX_NAME") + or index_name_for(corpus_sig, build_fp), + }, + "latency_ms": {}, + "recall": {}, + } + request.config._benchmark_results = results + return results + + +# --------------------------------------------------------------------------- +# Tests — pytest collects these in declaration order (measure → guard). +# Results serialization happens in conftest.pytest_sessionfinish, so the +# artifact is written even under -x/--maxfail, reordering plugins, or guard +# failures. Locally and on fork PRs the guards degrade gracefully (skip) +# when measurement data is missing. In trusted CI the guards FAIL on missing +# measurement data — a regression gate that silently passes without +# measurements is a green no-op — so a random-ordering plugin that runs a +# guard before the measurement tests will fail loudly there, by design. +# --------------------------------------------------------------------------- + + +class TestBenchmarkLatency: + """Measure end-to-end query latency over multiple rounds.""" + + def test_latency(self, moss_client, benchmark_results): + from moss import QueryOptions + + client, index_name = moss_client + latencies: list[float] = [] + + async def _measure(): + # Warmup + for _ in range(WARMUP_ROUNDS): + for q in QUERIES: + await client.query( + index_name, q, QueryOptions(top_k=TOP_K_LATENCY, alpha=1) + ) + + # Measured rounds + for _ in range(QUERY_ROUNDS): + for q in QUERIES: + start = time.perf_counter() + await client.query( + index_name, q, QueryOptions(top_k=TOP_K_LATENCY, alpha=1) + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + latencies.append(elapsed_ms) + + _run(_measure()) + + latencies.sort() + result = { + "p50": round(_percentile(latencies, 0.50), 3), + "p95": round(_percentile(latencies, 0.95), 3), + "p99": round(_percentile(latencies, 0.99), 3), + "mean": round(statistics.mean(latencies), 3), + "stdev": round(statistics.stdev(latencies), 3) if len(latencies) >= 2 else 0.0, + "count": len(latencies), + } + benchmark_results["latency_ms"] = result + + # Print for CI logs + print(f"\n Latency ({len(latencies)} measurements):") + print(f" P50 : {result['p50']:.3f} ms") + print(f" P95 : {result['p95']:.3f} ms") + print(f" P99 : {result['p99']:.3f} ms") + print(f" Mean : {result['mean']:.3f} ms") + print(f" Stdev: {result['stdev']:.3f} ms") + + +class TestBenchmarkRecall: + """Measure recall@k against pre-computed ground truth.""" + + def test_recall(self, moss_client, ground_truth, benchmark_results): + from moss import QueryOptions + + client, index_name = moss_client + + recall_at_5_scores: list[float] = [] + recall_at_10_scores: list[float] = [] + + async def _evaluate(): + for q in QUERIES: + expected_ids = ground_truth.get(q) + if not expected_ids: + # Silently skipping would shrink the evaluated set and + # inflate recall — fail loudly instead. + raise AssertionError( + f"Ground truth missing results for query {q!r}; " + "regenerate benchmarks/ci/ground_truth.json" + ) + + # recall@10 — fetch 10 results, also compute recall@5 + result = await client.query( + index_name, q, QueryOptions(top_k=TOP_K_RECALL_10, alpha=1) + ) + returned_ids = [doc.id for doc in result.docs] + + # recall@5 + expected_5 = set(expected_ids[:TOP_K_RECALL_5]) + returned_5 = set(returned_ids[:TOP_K_RECALL_5]) + if expected_5: + recall_at_5_scores.append( + len(expected_5 & returned_5) / len(expected_5) + ) + + # recall@10 + expected_10 = set(expected_ids[:TOP_K_RECALL_10]) + returned_10 = set(returned_ids[:TOP_K_RECALL_10]) + if expected_10: + recall_at_10_scores.append( + len(expected_10 & returned_10) / len(expected_10) + ) + + _run(_evaluate()) + + recall_5 = round(statistics.mean(recall_at_5_scores), 4) if recall_at_5_scores else 0.0 + recall_10 = round(statistics.mean(recall_at_10_scores), 4) if recall_at_10_scores else 0.0 + + benchmark_results["recall"] = { + "recall_at_5": recall_5, + "recall_at_10": recall_10, + "queries_evaluated": len(recall_at_5_scores), + } + + print(f"\n Recall ({len(recall_at_5_scores)} queries evaluated):") + print(f" Recall@5 : {recall_5:.4f}") + print(f" Recall@10 : {recall_10:.4f}") + + +class TestRegressionGuard: + """Compare current run against the checked-in baseline.""" + + def test_no_latency_regression(self, request, benchmark_results): + baseline_path = request.config.getoption("--baseline-file") + threshold = request.config.getoption("--latency-threshold") + + if not baseline_path: + pytest.skip("No baseline file provided — skipping regression check") + if not Path(baseline_path).exists(): + # An explicitly requested baseline that is absent is a config + # error, not an optional feature — never a green skip. + pytest.fail(f"--baseline-file was provided but does not exist: {baseline_path}") + + with open(baseline_path) as f: + baseline = json.load(f) + + baseline_p95 = baseline.get("latency_ms", {}).get("p95") + current_p95 = benchmark_results.get("latency_ms", {}).get("p95") + + if baseline_p95 is None or current_p95 is None: + # A baseline comparison was requested but there is nothing to + # compare: fine when measurement legitimately skipped (fork PR / + # local run without credentials), a red flag in trusted CI. + _missing_required_input( + "Latency measurement data missing — the latency test did not run" + ) + + _assert_baseline_compatible(baseline, benchmark_results) + + if baseline.get("latency_guard") == "unarmed": + # The checked-in placeholder declares itself unarmed: latency + # baselines must come from CI runners, and none has been captured + # yet. Skip loudly (recall is still guarded) instead of failing + # every trusted run until the first artifact lands. + if baseline_p95 != 0: + pytest.fail( + "baseline.json marks latency_guard as 'unarmed' but contains " + "a non-zero p95 — remove the latency_guard flag to arm the " + "guard." + ) + pytest.skip( + "LATENCY GUARD NOT ARMED — baseline.json is the explicit " + "placeholder (latency_guard: unarmed). To arm it: download the " + "benchmark-results- artifact from a trusted CI run and " + "commit it as benchmarks/ci/baseline.json (the artifact carries " + "no latency_guard flag, so committing it arms the guard)." + ) + + if baseline_p95 == 0: + # Zero without the explicit unarmed marker is a misconfigured + # baseline, not a placeholder — never a silent pass. + pytest.fail( + "Baseline p95 is zero but baseline.json does not declare " + "latency_guard: unarmed — the baseline is misconfigured. Commit " + "a CI-captured baseline or restore the explicit placeholder." + ) + + regression = (current_p95 - baseline_p95) / baseline_p95 + + print("\n Latency regression check:") + print(f" Baseline P95 : {baseline_p95:.3f} ms") + print(f" Current P95 : {current_p95:.3f} ms") + print(f" Change : {regression:+.1%}") + print(f" Threshold : {threshold:.0%}") + + assert regression <= threshold, ( + f"P95 latency regressed by {regression:+.1%} " + f"(baseline={baseline_p95:.3f}ms, current={current_p95:.3f}ms, " + f"threshold={threshold:.0%})" + ) + + def test_no_recall_regression(self, request, benchmark_results): + baseline_path = request.config.getoption("--baseline-file") + threshold = request.config.getoption("--recall-threshold") + + if not baseline_path: + pytest.skip("No baseline file provided — skipping regression check") + if not Path(baseline_path).exists(): + pytest.fail(f"--baseline-file was provided but does not exist: {baseline_path}") + + with open(baseline_path) as f: + baseline = json.load(f) + + # Guard every recall metric the suite records — checking only + # recall@5 would let a change that preserves the top 5 but drops + # documents ranked 6-10 pass while recall@10 regresses. + recall_pairs: dict[str, tuple[float, float]] = {} + for key in ("recall_at_5", "recall_at_10"): + baseline_val = baseline.get("recall", {}).get(key) + current_val = benchmark_results.get("recall", {}).get(key) + if baseline_val is None or current_val is None: + _missing_required_input( + f"Recall measurement data missing for {key} — " + "the recall test did not run" + ) + recall_pairs[key] = (baseline_val, current_val) + + _assert_baseline_compatible(baseline, benchmark_results) + + failures: list[str] = [] + print("\n Recall regression check:") + print(f" Threshold: {threshold:.4f}") + for key, (baseline_val, current_val) in recall_pairs.items(): + drop = baseline_val - current_val + print( + f" {key}: baseline={baseline_val:.4f} " + f"current={current_val:.4f} drop={drop:+.4f}" + ) + if drop > threshold: + failures.append( + f"{key} dropped by {drop:.4f} (baseline={baseline_val:.4f}, " + f"current={current_val:.4f}, threshold={threshold:.4f})" + ) + + assert not failures, "Recall regression: " + "; ".join(failures)