From be9d9f6ebb9f4216a29c0b965d15e77a16f388fc Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Fri, 26 Jun 2026 08:20:58 +0530 Subject: [PATCH 01/17] feat(moss-cli): add shell completions --- packages/moss-cli/CHANGELOG.md | 6 ++ packages/moss-cli/README.md | 25 ++++++ .../src/moss_cli/commands/completions.py | 56 ++++++++++++ .../moss-cli/src/moss_cli/commands/doc.py | 7 +- .../moss-cli/src/moss_cli/commands/index.py | 5 +- .../moss-cli/src/moss_cli/commands/search.py | 3 +- .../moss-cli/src/moss_cli/commands/sync.py | 3 +- packages/moss-cli/src/moss_cli/completion.py | 44 ++++++++++ packages/moss-cli/src/moss_cli/main.py | 2 + packages/moss-cli/tests/test_completions.py | 86 +++++++++++++++++++ 10 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 packages/moss-cli/src/moss_cli/commands/completions.py create mode 100644 packages/moss-cli/src/moss_cli/completion.py create mode 100644 packages/moss-cli/tests/test_completions.py diff --git a/packages/moss-cli/CHANGELOG.md b/packages/moss-cli/CHANGELOG.md index ba9f0fd6..6f989100 100644 --- a/packages/moss-cli/CHANGELOG.md +++ b/packages/moss-cli/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [Unreleased] + +- Shell completions: `moss completions bash` and `moss completions zsh` output + completion scripts covering commands, subcommands, and global flags, with + dynamic completion of index names + ## [0.1.0] - 2026-03-29 - **Initial release** — CLI wrapper for the Moss Python SDK (v1.0.0) diff --git a/packages/moss-cli/README.md b/packages/moss-cli/README.md index a6d0d20d..600d59df 100644 --- a/packages/moss-cli/README.md +++ b/packages/moss-cli/README.md @@ -172,6 +172,31 @@ moss profile list moss profile delete staging --force ``` +## Shell Completions + +Enable tab-completion for commands, subcommands, flags, and index names in Bash +and Zsh. `moss completions ` prints a completion script to stdout. + +```bash +# Bash — add to your shell startup file +moss completions bash >> ~/.bashrc + +# Zsh +moss completions zsh >> ~/.zshrc +``` + +Then restart your shell (or `source` the file) to activate it. Once enabled, +index-name arguments complete dynamically against your available indexes: + +```bash +moss query # lists your indexes +moss index delete sup +``` + +Dynamic index completion uses your resolved credentials (flags, env vars, or the +active profile). If credentials are missing or unreachable, completion simply +falls back to no suggestions rather than erroring. + ## Document File Format ### JSON (recommended) diff --git a/packages/moss-cli/src/moss_cli/commands/completions.py b/packages/moss-cli/src/moss_cli/commands/completions.py new file mode 100644 index 00000000..7c78d9a5 --- /dev/null +++ b/packages/moss-cli/src/moss_cli/commands/completions.py @@ -0,0 +1,56 @@ +"""moss completions command — output shell completion scripts.""" + +from __future__ import annotations + +from enum import Enum + +import typer + +from .. import output + +PROG_NAME = "moss" + + +class Shell(str, Enum): + bash = "bash" + zsh = "zsh" + + +def completions_command( + ctx: typer.Context, + shell: Shell = typer.Argument( + ..., help="Shell to generate the completion script for." + ), +) -> None: + """Output a shell completion script for Bash or Zsh. + + Tab-completion covers commands, subcommands, global flags, and index names. + + Bash: + + moss completions bash >> ~/.bashrc + + Zsh: + + moss completions zsh >> ~/.zshrc + + Then restart your shell (or 'source' the file) to activate it. + """ + json_mode = ctx.obj.get("json_output", False) if ctx.obj else False + + try: + from typer._completion_shared import get_completion_script + except Exception: # pragma: no cover - depends on Typer internals + output.print_error( + "Shell completion is unavailable in this Typer installation.", + json_mode, + ) + raise typer.Exit(1) + + complete_var = "_{}_COMPLETE".format(PROG_NAME.replace("-", "_").upper()) + script = get_completion_script( + prog_name=PROG_NAME, complete_var=complete_var, shell=shell.value + ) + # Emit the raw script with no Rich markup so it can be piped or redirected + # to a file verbatim. + typer.echo(script) diff --git a/packages/moss-cli/src/moss_cli/commands/doc.py b/packages/moss-cli/src/moss_cli/commands/doc.py index 99a8d8c1..b71059a3 100644 --- a/packages/moss-cli/src/moss_cli/commands/doc.py +++ b/packages/moss-cli/src/moss_cli/commands/doc.py @@ -11,6 +11,7 @@ from moss import MossClient, GetDocumentsOptions, MutationOptions from .. import output +from ..completion import complete_index_name from ..config import resolve_credentials from ..documents import load_documents from ..job_waiter import wait_for_job @@ -29,7 +30,7 @@ def _client(ctx: typer.Context) -> MossClient: @doc_app.command(name="add") def add( ctx: typer.Context, - index_name: str = typer.Argument(..., help="Index name"), + index_name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), file: str = typer.Option(..., "--file", "-f", help="Path to JSON/CSV document file, or '-' for stdin"), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" @@ -64,7 +65,7 @@ def add( @doc_app.command(name="delete") def delete( ctx: typer.Context, - index_name: str = typer.Argument(..., help="Index name"), + index_name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), ids: str = typer.Option(..., "--ids", "-i", help="Comma-separated document IDs"), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" @@ -97,7 +98,7 @@ def delete( @doc_app.command(name="get") def get( ctx: typer.Context, - index_name: str = typer.Argument(..., help="Index name"), + index_name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), ids: Optional[str] = typer.Option(None, "--ids", "-i", help="Comma-separated document IDs (omit for all)"), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" diff --git a/packages/moss-cli/src/moss_cli/commands/index.py b/packages/moss-cli/src/moss_cli/commands/index.py index 6145cce9..3f255187 100644 --- a/packages/moss-cli/src/moss_cli/commands/index.py +++ b/packages/moss-cli/src/moss_cli/commands/index.py @@ -11,6 +11,7 @@ from moss import MossClient from .. import output +from ..completion import complete_index_name from ..config import resolve_credentials from ..documents import load_documents from ..job_waiter import wait_for_job @@ -75,7 +76,7 @@ def list_indexes( @index_app.command(name="get") def get( ctx: typer.Context, - name: str = typer.Argument(..., help="Index name"), + name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" ), @@ -92,7 +93,7 @@ def get( @index_app.command(name="delete") def delete( ctx: typer.Context, - name: str = typer.Argument(..., help="Index name"), + name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" ), diff --git a/packages/moss-cli/src/moss_cli/commands/search.py b/packages/moss-cli/src/moss_cli/commands/search.py index 11c09f61..5d241656 100644 --- a/packages/moss-cli/src/moss_cli/commands/search.py +++ b/packages/moss-cli/src/moss_cli/commands/search.py @@ -13,6 +13,7 @@ from moss import MossClient, QueryOptions from .. import output +from ..completion import complete_index_name from ..config import resolve_credentials console = Console() @@ -47,7 +48,7 @@ def _parse_set_command(line: str) -> tuple[Optional[str], Optional[str]]: def query_command( ctx: typer.Context, - index_name: str = typer.Argument(..., help="Index name"), + index_name: str = typer.Argument(..., help="Index name", autocompletion=complete_index_name), query_text: Optional[str] = typer.Argument(None, help="Search query (reads from stdin if omitted)"), profile: Optional[str] = typer.Option( None, "--profile", help="Credential profile name" diff --git a/packages/moss-cli/src/moss_cli/commands/sync.py b/packages/moss-cli/src/moss_cli/commands/sync.py index 18f0da43..37db5520 100644 --- a/packages/moss-cli/src/moss_cli/commands/sync.py +++ b/packages/moss-cli/src/moss_cli/commands/sync.py @@ -12,6 +12,7 @@ from moss import MossClient, MutationOptions from .. import output +from ..completion import complete_index_name from ..config import resolve_credentials from ..documents import load_documents from ..job_waiter import wait_for_job @@ -66,7 +67,7 @@ async def _upsert_file( def sync_command( ctx: typer.Context, directory: Path = typer.Argument(..., help="Directory containing document files"), - index_name: str = typer.Argument(..., help="Index to upsert documents into"), + index_name: str = typer.Argument(..., help="Index to upsert documents into", autocompletion=complete_index_name), watch: bool = typer.Option(False, "--watch", "-w", help="Keep watching for file changes"), ext: str = typer.Option("json,jsonl,csv", "--ext", help="Comma-separated file extensions to include"), scan_interval: float = typer.Option(2.0, "--scan-interval", help="Seconds between directory scans (watch mode)"), diff --git a/packages/moss-cli/src/moss_cli/completion.py b/packages/moss-cli/src/moss_cli/completion.py new file mode 100644 index 00000000..b62c526a --- /dev/null +++ b/packages/moss-cli/src/moss_cli/completion.py @@ -0,0 +1,44 @@ +"""Shared shell-completion helpers for dynamic value completion.""" + +from __future__ import annotations + +import asyncio +from typing import List + +import typer + +from .config import resolve_credentials + + +def complete_index_name( + ctx: typer.Context, args: List[str], incomplete: str +) -> List[str]: + """Autocompletion callback that lists the user's index names. + + The shell invokes this while completing an index-name argument. It resolves + credentials the same way commands do (flags > env vars > active profile) and + lists the available indexes. + + It is intentionally best-effort: any failure (missing credentials, network + error, or the SDK not being importable) returns no completions so the shell + never errors out or blocks. Typer filters the returned names against the + text typed so far, so we return the full list. + """ + try: + from moss import MossClient # lazy import; the SDK is heavy + + project_id = None + project_key = None + profile = None + root = ctx.find_root() if ctx is not None else None + if root is not None and root.params: + project_id = root.params.get("project_id") + project_key = root.params.get("project_key") + profile = root.params.get("profile") + + pid, pkey = resolve_credentials(project_id, project_key, profile) + client = MossClient(pid, pkey) + indexes = asyncio.run(client.list_indexes()) + return [idx.name for idx in indexes if getattr(idx, "name", None)] + except Exception: + return [] diff --git a/packages/moss-cli/src/moss_cli/main.py b/packages/moss-cli/src/moss_cli/main.py index 2f1bb8f0..eae0ef50 100644 --- a/packages/moss-cli/src/moss_cli/main.py +++ b/packages/moss-cli/src/moss_cli/main.py @@ -7,6 +7,7 @@ import typer +from .commands.completions import completions_command from .commands.doc import doc_app from .commands.index import index_app from .commands.init_cmd import init_command @@ -37,6 +38,7 @@ app.command(name="version")(version_command) app.command(name="validate")(validate_command) app.command(name="sync")(sync_command) +app.command(name="completions")(completions_command) @app.callback() diff --git a/packages/moss-cli/tests/test_completions.py b/packages/moss-cli/tests/test_completions.py new file mode 100644 index 00000000..b7b2c185 --- /dev/null +++ b/packages/moss-cli/tests/test_completions.py @@ -0,0 +1,86 @@ +from typer.testing import CliRunner + +from moss_cli import completion +from moss_cli.main import app + +runner = CliRunner() + + +def test_completions_bash_outputs_script(): + result = runner.invoke(app, ["completions", "bash"]) + + assert result.exit_code == 0 + # Click/Typer bash completion script markers. + assert "_moss_completion" in result.stdout + assert "_MOSS_COMPLETE=complete_bash" in result.stdout + assert "complete -o default -F _moss_completion moss" in result.stdout + + +def test_completions_zsh_outputs_script(): + result = runner.invoke(app, ["completions", "zsh"]) + + assert result.exit_code == 0 + assert "#compdef moss" in result.stdout + assert "_MOSS_COMPLETE=complete_zsh" in result.stdout + assert "compdef _moss_completion moss" in result.stdout + + +def test_completions_rejects_unsupported_shell(): + result = runner.invoke(app, ["completions", "fish"]) + + # Typer validates the Shell enum and exits with a usage error. + assert result.exit_code != 0 + + +def test_complete_index_name_lists_indexes(monkeypatch): + class FakeIndex: + def __init__(self, name): + self.name = name + + class FakeClient: + def __init__(self, project_id, project_key): + pass + + async def list_indexes(self): + return [FakeIndex("alpha"), FakeIndex("beta"), FakeIndex("gamma")] + + monkeypatch.setenv("MOSS_PROJECT_ID", "pid") + monkeypatch.setenv("MOSS_PROJECT_KEY", "pkey") + # complete_index_name imports MossClient from the moss module lazily. + import moss + + monkeypatch.setattr(moss, "MossClient", FakeClient) + + names = completion.complete_index_name(None, [], "") + assert names == ["alpha", "beta", "gamma"] + + +def test_complete_index_name_returns_empty_on_error(monkeypatch): + class BrokenClient: + def __init__(self, project_id, project_key): + raise RuntimeError("boom") + + monkeypatch.setenv("MOSS_PROJECT_ID", "pid") + monkeypatch.setenv("MOSS_PROJECT_KEY", "pkey") + import moss + + monkeypatch.setattr(moss, "MossClient", BrokenClient) + + # Any failure must yield no completions rather than raising. + assert completion.complete_index_name(None, [], "") == [] + + +def test_complete_index_name_handles_missing_credentials(monkeypatch): + # No credentials available anywhere -> resolve_credentials raises -> []. + monkeypatch.delenv("MOSS_PROJECT_ID", raising=False) + monkeypatch.delenv("MOSS_PROJECT_KEY", raising=False) + monkeypatch.delenv("MOSS_PROFILE", raising=False) + monkeypatch.setattr(completion, "resolve_credentials", _raise_bad_params) + + assert completion.complete_index_name(None, [], "") == [] + + +def _raise_bad_params(*args, **kwargs): + import typer + + raise typer.BadParameter("missing creds") From 8e4937b7b49a35e1bef762eb2bcaea74a59bc4c1 Mon Sep 17 00:00:00 2001 From: Sravan Avvaru <81159574+Sravan1011@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:05:28 +0530 Subject: [PATCH 02/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../moss-cli/src/moss_cli/commands/completions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/moss-cli/src/moss_cli/commands/completions.py b/packages/moss-cli/src/moss_cli/commands/completions.py index 7c78d9a5..d110ab01 100644 --- a/packages/moss-cli/src/moss_cli/commands/completions.py +++ b/packages/moss-cli/src/moss_cli/commands/completions.py @@ -39,8 +39,17 @@ def completions_command( json_mode = ctx.obj.get("json_output", False) if ctx.obj else False try: - from typer._completion_shared import get_completion_script - except Exception: # pragma: no cover - depends on Typer internals + # Prefer a public API when available. + from typer.main import get_completion_script # type: ignore[attr-defined] + except Exception: # pragma: no cover + try: + from typer._completion_shared import get_completion_script # type: ignore + except Exception: # pragma: no cover - depends on Typer installation + output.print_error( + "Shell completion is unavailable in this Typer installation.", + json_mode, + ) + raise typer.Exit(1) output.print_error( "Shell completion is unavailable in this Typer installation.", json_mode, From f4732a1e5878e8c6427bc4bb185f53815ba4c261 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Mon, 20 Jul 2026 11:15:34 +0530 Subject: [PATCH 03/17] Add CI benchmark suite for latency and recall regression tracking (#435) Adds an automated benchmark harness under benchmarks/ci/ that runs on every push/PR to main, records p50/p95/p99 latency and recall@5/@10 against a fixed 1K-doc corpus, and fails the build when results regress past configurable thresholds versus a checked-in baseline. - test_bench_ci_moss.py: pytest suite (latency, recall, regression guard, JSON results writer) - conftest.py: CLI flags for output path, baseline file, and thresholds - generate_ground_truth.py + ground_truth.json: pre-computed expected results per query (top_k=50, moss-minilm) - baseline.json: placeholder baseline; latency guard skips until a real baseline is captured from CI hardware - .github/workflows/benchmark.yml: CI job with artifact upload and a manual update_baseline dispatch input Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 63 ++ .gitignore | 3 +- AGENTS.md | 7 + benchmarks/.gitignore | 2 + benchmarks/README.md | 11 +- benchmarks/ci/README.md | 91 +++ benchmarks/ci/baseline.json | 24 + benchmarks/ci/conftest.py | 36 ++ benchmarks/ci/generate_ground_truth.py | 114 ++++ benchmarks/ci/ground_truth.json | 788 +++++++++++++++++++++++++ benchmarks/ci/requirements.txt | 6 + benchmarks/ci/test_bench_ci_moss.py | 374 ++++++++++++ 12 files changed, 1517 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmarks/ci/README.md create mode 100644 benchmarks/ci/baseline.json create mode 100644 benchmarks/ci/conftest.py create mode 100644 benchmarks/ci/generate_ground_truth.py create mode 100644 benchmarks/ci/ground_truth.json create mode 100644 benchmarks/ci/requirements.txt create mode 100644 benchmarks/ci/test_bench_ci_moss.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..c6217009 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,63 @@ +name: Benchmark + +permissions: + contents: read + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + inputs: + update_baseline: + description: 'Update baseline.json with current results' + type: boolean + default: false + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r benchmarks/ci/requirements.txt + + - name: Run benchmark suite + env: + MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} + MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} + run: | + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json \ + --latency-threshold=0.20 \ + --recall-threshold=0.05 + + - name: Upload results artifact + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.sha }} + path: benchmark_results.json + retention-days: 90 + if: always() + + - name: Update baseline (manual trigger only) + if: >- + github.event_name == 'workflow_dispatch' && + github.event.inputs.update_baseline == 'true' + run: | + echo "Copying benchmark_results.json → benchmarks/ci/baseline.json" + cp benchmark_results.json benchmarks/ci/baseline.json + echo "Updated baseline:" + cat benchmarks/ci/baseline.json + echo "" + echo "NOTE: To persist this change, commit and push benchmarks/ci/baseline.json" diff --git a/.gitignore b/.gitignore index b5028c58..9973c0b2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ node_modules/ dist/ .next/ coverage/ -.venv/ +.venv*/ env/ venv/ *.log @@ -27,3 +27,4 @@ skills-lock.json # Swift / SwiftPM build artifacts .build/ +benchmark_results.json diff --git a/AGENTS.md b/AGENTS.md index a4291351..fd69d9b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,3 +268,10 @@ The `.github/workflows/ci.yml` pipeline runs on push to `main` and on PRs: - `python-sdk-test` — matrix over Python 3.10–3.14 - `javascript-lint` — eslint - Separate release workflows publish to PyPI / npm on tagged releases + +The `.github/workflows/benchmark.yml` runs on push to `main` and on PRs: +- Runs the CI benchmark suite (`benchmarks/ci/`) measuring p50/p95/p99 latency and recall@k +- Compares against `benchmarks/ci/baseline.json` and fails on regressions exceeding thresholds +- Uploads `benchmark_results.json` as a workflow artifact for each run +- Supports `workflow_dispatch` with `update_baseline` input to refresh the baseline + diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore index 6dc235bd..156301f4 100644 --- a/benchmarks/.gitignore +++ b/benchmarks/.gitignore @@ -1,3 +1,5 @@ .env __pycache__/ embedding_server/__pycache__/ +ci/__pycache__/ +benchmark_results.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 378c3a99..1831c3e2 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -77,4 +77,13 @@ EMBEDDING_DIMENSION=768 - **Queries**: 15 diverse search queries - **Warmup**: 3 rounds (excluded from measurements) - **Measured**: 50 rounds x 15 queries = 750 measurements per system -- **top_k**: 5 \ No newline at end of file +- **top_k**: 5 + +## CI Benchmark Suite + +For automated regression testing in CI, see [`benchmarks/ci/`](ci/). +This suite runs on every push to `main` and on PRs, tracking p50/p95/p99 +latency and recall@k per commit. It compares against a checked-in baseline +and fails the build if regressions exceed configurable thresholds. + +See [`benchmarks/ci/README.md`](ci/README.md) for full documentation. \ No newline at end of file diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md new file mode 100644 index 00000000..d04a90e2 --- /dev/null +++ b/benchmarks/ci/README.md @@ -0,0 +1,91 @@ +# CI Benchmark Suite — Latency & Recall Regression Guard + +Automated benchmark harness that runs on every push/PR to `main` and catches +performance regressions before they ship. + +## What it measures + +| Metric | Description | +|--------|-------------| +| **P50 / P95 / P99 latency** | End-to-end query latency (embedding + search) in ms | +| **Mean / Stdev** | Average and standard deviation of latency | +| **Recall@5** | Fraction of ground-truth top-5 docs returned in top 5 | +| **Recall@10** | Fraction of ground-truth top-10 docs returned in top 10 | + +All measurements use the Moss built-in embedding model (`moss-minilm`) with a +1,000-document subset of the benchmark corpus for CI speed. + +## Quick start + +### Prerequisites + +Set `MOSS_PROJECT_ID` and `MOSS_PROJECT_KEY` in your environment (or in a +`.env` file). + +### Run locally + +```bash +# Install dependencies +pip install -r benchmarks/ci/requirements.txt + +# Run the full suite +pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json + +# Skip regression checks (no baseline comparison) +pytest benchmarks/ci/ -v --benchmark-output=benchmark_results.json +``` + +### Regenerate ground truth + +Run this when the index data or model changes: + +```bash +python benchmarks/ci/generate_ground_truth.py +``` + +This queries Moss with `top_k=50` for each benchmark query and writes the +expected document IDs to `ground_truth.json`. Commit the updated file. + +## How regression detection works + +The harness compares the current run's metrics against `baseline.json`: + +- **Latency**: Fails if P95 increases by more than the threshold (default 20%) +- **Recall**: Fails if Recall@5 drops by more than the threshold (default 5pp) + +Thresholds are configurable via CLI flags: + +```bash +pytest benchmarks/ci/ -v \ + --latency-threshold=0.15 \ # 15% max P95 regression + --recall-threshold=0.03 # 3pp max recall drop +``` + +## Updating the baseline + +After a legitimate performance change (e.g., model upgrade, index config +change), update the baseline: + +1. **Via GitHub Actions**: Trigger the `Benchmark` workflow manually with + `update_baseline=true` +2. **Manually**: Copy a CI run's `benchmark_results.json` artifact to + `benchmarks/ci/baseline.json` and commit + +## CI integration + +The benchmark runs as a GitHub Actions job (`.github/workflows/benchmark.yml`). +Results are uploaded as artifacts named `benchmark-results-` and are +available for download from the Actions tab. + +## File overview + +| File | Purpose | +|------|---------| +| `test_bench_ci_moss.py` | Main test module (latency, recall, regression guard) | +| `conftest.py` | Pytest CLI flags | +| `generate_ground_truth.py` | One-time ground truth generator | +| `ground_truth.json` | Pre-computed expected results per query | +| `baseline.json` | Performance baseline for regression checks | +| `requirements.txt` | Python dependencies | diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json new file mode 100644 index 00000000..4471b2f8 --- /dev/null +++ b/benchmarks/ci/baseline.json @@ -0,0 +1,24 @@ +{ + "commit": "initial", + "timestamp": "2025-07-19T00:00:00+00:00", + "_note": "Placeholder baseline — update via 'workflow_dispatch' or copy a CI artifact.", + "latency_ms": { + "p50": 0, + "p95": 0, + "p99": 0, + "mean": 0, + "stdev": 0, + "count": 0 + }, + "recall": { + "recall_at_5": 0, + "recall_at_10": 0, + "queries_evaluated": 0 + }, + "config": { + "doc_count": 1000, + "query_rounds": 20, + "warmup_rounds": 3, + "top_k_latency": 5 + } +} diff --git a/benchmarks/ci/conftest.py b/benchmarks/ci/conftest.py new file mode 100644 index 00000000..3d0e1f1a --- /dev/null +++ b/benchmarks/ci/conftest.py @@ -0,0 +1,36 @@ +"""Pytest configuration for the CI benchmark suite. + +Adds custom CLI flags so the harness can be invoked as a standard pytest +run with configurable output paths and regression thresholds. +""" + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("benchmark", "CI benchmark options") + group.addoption( + "--benchmark-output", + default="benchmark_results.json", + help="Path to write the JSON results file (default: benchmark_results.json)", + ) + group.addoption( + "--baseline-file", + default=None, + help="Path to baseline JSON for regression comparison. " + "If not provided, regression checks are skipped.", + ) + group.addoption( + "--latency-threshold", + type=float, + default=0.20, + help="Max allowed fractional increase in p95 latency vs baseline " + "(default: 0.20 = 20%%)", + ) + group.addoption( + "--recall-threshold", + type=float, + default=0.05, + help="Max allowed absolute decrease in recall@k vs baseline " + "(default: 0.05 = 5 percentage points)", + ) diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py new file mode 100644 index 00000000..817f3986 --- /dev/null +++ b/benchmarks/ci/generate_ground_truth.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Generate ground truth for CI benchmark recall computation. + +Queries the Moss index with a large top_k and records the returned document +IDs as the "expected" relevant set for each benchmark query. Run this once +(or whenever the index/model changes) and commit the output. + +Usage:: + + # Ensure MOSS_PROJECT_ID and MOSS_PROJECT_KEY are set + python benchmarks/ci/generate_ground_truth.py + +Output is written to ``benchmarks/ci/ground_truth.json``. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + +# Re-use the same query set as the CI benchmark. +QUERIES = [ + "neural network training data", + "anomaly detection patterns", + "computer vision image processing", + "natural language processing", + "reinforcement learning rewards", + "transfer learning pretrained models", + "distributed computing systems", + "cryptographic data encryption", + "database indexing performance", + "knowledge graph entities", + "generative adversarial networks", + "attention mechanism transformers", + "dimensionality reduction compression", + "federated learning privacy", + "stream processing pipelines", +] + +# Fetch a generous top_k so recall@5 and recall@10 can be evaluated +# against a superset of relevant results. +GROUND_TRUTH_TOP_K = 50 + + +async def main() -> None: + from moss import MossClient, DocumentInfo, QueryOptions + + project_id = os.getenv("MOSS_PROJECT_ID") + project_key = os.getenv("MOSS_PROJECT_KEY") + index_name = os.getenv("MOSS_INDEX_NAME", "benchmark-ci") + + if not project_id or not project_key: + print("Error: MOSS_PROJECT_ID and MOSS_PROJECT_KEY must be set.") + sys.exit(1) + + client = MossClient(project_id, project_key) + + # Ensure the index exists (create with a 1K subset if needed). + try: + await client.get_index(index_name) + print(f"Using existing index '{index_name}'") + except Exception: + corpus_path = Path(__file__).resolve().parent.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + print(f"Error: Corpus file not found: {corpus_path}") + sys.exit(1) + with open(corpus_path) as f: + all_docs = json.load(f) + docs = [ + DocumentInfo(id=d["id"], text=d["text"], metadata=d.get("metadata")) + for d in all_docs[:1000] + ] + result = await client.create_index(index_name, docs, "moss-minilm") + print(f"Created index '{index_name}' with {result.doc_count} docs") + + await client.load_index(index_name) + + # Query each benchmark query with a large top_k. + ground_truth: dict[str, list[str]] = {} + for q in QUERIES: + result = await client.query( + index_name, + q, + QueryOptions(top_k=GROUND_TRUTH_TOP_K, alpha=1), + ) + doc_ids = [doc.id for doc in result.docs] + ground_truth[q] = doc_ids + print(f" '{q}' → {len(doc_ids)} results") + + output = { + "model": "moss-minilm", + "top_k": GROUND_TRUTH_TOP_K, + "index_name": index_name, + "doc_count": 1000, + "queries": ground_truth, + } + + output_path = Path(__file__).resolve().parent / "ground_truth.json" + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + + print(f"\nGround truth written to: {output_path}") + print(f"Queries: {len(ground_truth)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/ci/ground_truth.json b/benchmarks/ci/ground_truth.json new file mode 100644 index 00000000..76d2e9a9 --- /dev/null +++ b/benchmarks/ci/ground_truth.json @@ -0,0 +1,788 @@ +{ + "model": "moss-minilm", + "top_k": 50, + "index_name": "benchmark-ci", + "doc_count": 1000, + "queries": { + "neural network training data": [ + "doc_113", + "doc_877", + "doc_91", + "doc_984", + "doc_200", + "doc_160", + "doc_946", + "doc_593", + "doc_349", + "doc_536", + "doc_325", + "doc_763", + "doc_23", + "doc_297", + "doc_460", + "doc_601", + "doc_865", + "doc_965", + "doc_62", + "doc_698", + "doc_128", + "doc_715", + "doc_455", + "doc_992", + "doc_229", + "doc_215", + "doc_447", + "doc_578", + "doc_243", + "doc_360", + "doc_510", + "doc_675", + "doc_157", + "doc_602", + "doc_794", + "doc_202", + "doc_824", + "doc_493", + "doc_503", + "doc_210", + "doc_641", + "doc_872", + "doc_633", + "doc_472", + "doc_6", + "doc_803", + "doc_329", + "doc_971", + "doc_350", + "doc_500" + ], + "anomaly detection patterns": [ + "doc_267", + "doc_249", + "doc_345", + "doc_700", + "doc_289", + "doc_806", + "doc_154", + "doc_929", + "doc_837", + "doc_612", + "doc_373", + "doc_120", + "doc_242", + "doc_896", + "doc_853", + "doc_495", + "doc_397", + "doc_854", + "doc_873", + "doc_861", + "doc_563", + "doc_170", + "doc_487", + "doc_855", + "doc_766", + "doc_13", + "doc_377", + "doc_758", + "doc_827", + "doc_735", + "doc_5", + "doc_57", + "doc_423", + "doc_381", + "doc_871", + "doc_471", + "doc_109", + "doc_180", + "doc_783", + "doc_643", + "doc_327", + "doc_459", + "doc_211", + "doc_507", + "doc_682", + "doc_92", + "doc_438", + "doc_58", + "doc_828", + "doc_29" + ], + "computer vision image processing": [ + "doc_165", + "doc_847", + "doc_878", + "doc_388", + "doc_843", + "doc_597", + "doc_80", + "doc_89", + "doc_71", + "doc_218", + "doc_19", + "doc_257", + "doc_129", + "doc_0", + "doc_677", + "doc_755", + "doc_124", + "doc_717", + "doc_548", + "doc_748", + "doc_805", + "doc_489", + "doc_366", + "doc_228", + "doc_830", + "doc_598", + "doc_520", + "doc_769", + "doc_815", + "doc_191", + "doc_684", + "doc_709", + "doc_193", + "doc_371", + "doc_547", + "doc_626", + "doc_690", + "doc_106", + "doc_294", + "doc_640", + "doc_3", + "doc_913", + "doc_33", + "doc_661", + "doc_248", + "doc_760", + "doc_721", + "doc_741", + "doc_139", + "doc_729" + ], + "natural language processing": [ + "doc_114", + "doc_278", + "doc_317", + "doc_181", + "doc_968", + "doc_943", + "doc_161", + "doc_112", + "doc_355", + "doc_32", + "doc_281", + "doc_7", + "doc_208", + "doc_30", + "doc_733", + "doc_178", + "doc_958", + "doc_988", + "doc_797", + "doc_978", + "doc_141", + "doc_670", + "doc_912", + "doc_594", + "doc_619", + "doc_867", + "doc_163", + "doc_879", + "doc_998", + "doc_826", + "doc_359", + "doc_589", + "doc_162", + "doc_842", + "doc_506", + "doc_546", + "doc_851", + "doc_93", + "doc_457", + "doc_502", + "doc_753", + "doc_901", + "doc_718", + "doc_26", + "doc_236", + "doc_754", + "doc_39", + "doc_949", + "doc_596", + "doc_868" + ], + "reinforcement learning rewards": [ + "doc_175", + "doc_318", + "doc_292", + "doc_813", + "doc_960", + "doc_846", + "doc_751", + "doc_60", + "doc_209", + "doc_955", + "doc_973", + "doc_911", + "doc_96", + "doc_707", + "doc_585", + "doc_897", + "doc_461", + "doc_577", + "doc_639", + "doc_644", + "doc_221", + "doc_655", + "doc_391", + "doc_818", + "doc_213", + "doc_850", + "doc_785", + "doc_917", + "doc_953", + "doc_320", + "doc_263", + "doc_683", + "doc_538", + "doc_942", + "doc_63", + "doc_449", + "doc_674", + "doc_557", + "doc_549", + "doc_492", + "doc_212", + "doc_342", + "doc_491", + "doc_584", + "doc_991", + "doc_386", + "doc_857", + "doc_713", + "doc_50", + "doc_952" + ], + "transfer learning pretrained models": [ + "doc_273", + "doc_339", + "doc_857", + "doc_108", + "doc_841", + "doc_50", + "doc_203", + "doc_177", + "doc_889", + "doc_246", + "doc_45", + "doc_386", + "doc_398", + "doc_991", + "doc_704", + "doc_527", + "doc_952", + "doc_909", + "doc_903", + "doc_197", + "doc_316", + "doc_587", + "doc_951", + "doc_895", + "doc_916", + "doc_519", + "doc_840", + "doc_713", + "doc_454", + "doc_521", + "doc_173", + "doc_115", + "doc_736", + "doc_255", + "doc_156", + "doc_408", + "doc_352", + "doc_580", + "doc_131", + "doc_574", + "doc_653", + "doc_504", + "doc_526", + "doc_405", + "doc_559", + "doc_323", + "doc_47", + "doc_74", + "doc_792", + "doc_225" + ], + "distributed computing systems": [ + "doc_479", + "doc_306", + "doc_940", + "doc_933", + "doc_997", + "doc_514", + "doc_511", + "doc_98", + "doc_849", + "doc_607", + "doc_85", + "doc_714", + "doc_369", + "doc_224", + "doc_277", + "doc_76", + "doc_994", + "doc_79", + "doc_599", + "doc_341", + "doc_146", + "doc_266", + "doc_782", + "doc_40", + "doc_908", + "doc_779", + "doc_227", + "doc_222", + "doc_543", + "doc_832", + "doc_64", + "doc_392", + "doc_919", + "doc_795", + "doc_443", + "doc_880", + "doc_15", + "doc_790", + "doc_608", + "doc_435", + "doc_615", + "doc_706", + "doc_614", + "doc_250", + "doc_192", + "doc_650", + "doc_446", + "doc_368", + "doc_560", + "doc_750" + ], + "cryptographic data encryption": [ + "doc_364", + "doc_336", + "doc_2", + "doc_486", + "doc_477", + "doc_298", + "doc_46", + "doc_312", + "doc_233", + "doc_107", + "doc_335", + "doc_31", + "doc_164", + "doc_689", + "doc_836", + "doc_666", + "doc_54", + "doc_712", + "doc_442", + "doc_490", + "doc_230", + "doc_286", + "doc_989", + "doc_483", + "doc_185", + "doc_8", + "doc_976", + "doc_948", + "doc_571", + "doc_656", + "doc_132", + "doc_282", + "doc_12", + "doc_110", + "doc_309", + "doc_950", + "doc_605", + "doc_699", + "doc_972", + "doc_463", + "doc_635", + "doc_378", + "doc_111", + "doc_431", + "doc_413", + "doc_923", + "doc_217", + "doc_881", + "doc_41", + "doc_226" + ], + "database indexing performance": [ + "doc_83", + "doc_475", + "doc_910", + "doc_253", + "doc_182", + "doc_647", + "doc_474", + "doc_637", + "doc_37", + "doc_696", + "doc_367", + "doc_102", + "doc_570", + "doc_152", + "doc_703", + "doc_321", + "doc_874", + "doc_133", + "doc_284", + "doc_441", + "doc_351", + "doc_436", + "doc_234", + "doc_702", + "doc_488", + "doc_767", + "doc_412", + "doc_362", + "doc_275", + "doc_216", + "doc_532", + "doc_432", + "doc_376", + "doc_928", + "doc_18", + "doc_49", + "doc_153", + "doc_694", + "doc_123", + "doc_743", + "doc_802", + "doc_95", + "doc_415", + "doc_970", + "doc_544", + "doc_926", + "doc_150", + "doc_937", + "doc_648", + "doc_44" + ], + "knowledge graph entities": [ + "doc_370", + "doc_810", + "doc_168", + "doc_609", + "doc_287", + "doc_610", + "doc_247", + "doc_894", + "doc_196", + "doc_380", + "doc_481", + "doc_875", + "doc_272", + "doc_957", + "doc_869", + "doc_775", + "doc_777", + "doc_900", + "doc_723", + "doc_402", + "doc_967", + "doc_9", + "doc_793", + "doc_244", + "doc_220", + "doc_28", + "doc_685", + "doc_409", + "doc_669", + "doc_884", + "doc_800", + "doc_975", + "doc_737", + "doc_924", + "doc_934", + "doc_705", + "doc_856", + "doc_465", + "doc_167", + "doc_796", + "doc_961", + "doc_764", + "doc_542", + "doc_556", + "doc_920", + "doc_87", + "doc_888", + "doc_280", + "doc_732", + "doc_466" + ], + "generative adversarial networks": [ + "doc_265", + "doc_350", + "doc_20", + "doc_67", + "doc_985", + "doc_340", + "doc_268", + "doc_659", + "doc_918", + "doc_625", + "doc_784", + "doc_145", + "doc_421", + "doc_290", + "doc_664", + "doc_899", + "doc_892", + "doc_291", + "doc_261", + "doc_130", + "doc_983", + "doc_528", + "doc_319", + "doc_823", + "doc_679", + "doc_588", + "doc_839", + "doc_931", + "doc_329", + "doc_344", + "doc_870", + "doc_332", + "doc_627", + "doc_353", + "doc_631", + "doc_375", + "doc_657", + "doc_523", + "doc_499", + "doc_788", + "doc_572", + "doc_693", + "doc_576", + "doc_403", + "doc_555", + "doc_890", + "doc_256", + "doc_840", + "doc_108", + "doc_713" + ], + "attention mechanism transformers": [ + "doc_428", + "doc_756", + "doc_384", + "doc_906", + "doc_21", + "doc_358", + "doc_907", + "doc_35", + "doc_845", + "doc_822", + "doc_382", + "doc_59", + "doc_692", + "doc_862", + "doc_687", + "doc_620", + "doc_658", + "doc_834", + "doc_866", + "doc_636", + "doc_36", + "doc_804", + "doc_389", + "doc_691", + "doc_104", + "doc_237", + "doc_680", + "doc_765", + "doc_407", + "doc_84", + "doc_184", + "doc_979", + "doc_662", + "doc_761", + "doc_628", + "doc_885", + "doc_665", + "doc_781", + "doc_245", + "doc_603", + "doc_533", + "doc_177", + "doc_890", + "doc_385", + "doc_916", + "doc_952", + "doc_580", + "doc_895", + "doc_197", + "doc_108" + ], + "dimensionality reduction compression": [ + "doc_140", + "doc_95", + "doc_119", + "doc_544", + "doc_343", + "doc_357", + "doc_554", + "doc_445", + "doc_395", + "doc_937", + "doc_334", + "doc_970", + "doc_678", + "doc_11", + "doc_101", + "doc_155", + "doc_618", + "doc_4", + "doc_10", + "doc_150", + "doc_926", + "doc_44", + "doc_194", + "doc_982", + "doc_333", + "doc_757", + "doc_668", + "doc_305", + "doc_478", + "doc_427", + "doc_55", + "doc_914", + "doc_524", + "doc_464", + "doc_902", + "doc_776", + "doc_190", + "doc_759", + "doc_993", + "doc_774", + "doc_72", + "doc_770", + "doc_819", + "doc_821", + "doc_219", + "doc_681", + "doc_590", + "doc_434", + "doc_648", + "doc_568" + ], + "federated learning privacy": [ + "doc_887", + "doc_337", + "doc_829", + "doc_347", + "doc_966", + "doc_716", + "doc_383", + "doc_977", + "doc_728", + "doc_835", + "doc_16", + "doc_354", + "doc_264", + "doc_251", + "doc_426", + "doc_518", + "doc_799", + "doc_962", + "doc_126", + "doc_143", + "doc_974", + "doc_322", + "doc_260", + "doc_673", + "doc_125", + "doc_254", + "doc_922", + "doc_905", + "doc_747", + "doc_201", + "doc_749", + "doc_515", + "doc_223", + "doc_604", + "doc_414", + "doc_817", + "doc_135", + "doc_745", + "doc_387", + "doc_169", + "doc_206", + "doc_65", + "doc_801", + "doc_271", + "doc_762", + "doc_573", + "doc_326", + "doc_622", + "doc_606", + "doc_531" + ], + "stream processing pipelines": [ + "doc_560", + "doc_505", + "doc_315", + "doc_250", + "doc_410", + "doc_663", + "doc_204", + "doc_848", + "doc_446", + "doc_708", + "doc_935", + "doc_558", + "doc_43", + "doc_750", + "doc_615", + "doc_509", + "doc_48", + "doc_53", + "doc_893", + "doc_308", + "doc_614", + "doc_959", + "doc_706", + "doc_241", + "doc_551", + "doc_686", + "doc_1", + "doc_195", + "doc_650", + "doc_778", + "doc_17", + "doc_368", + "doc_734", + "doc_986", + "doc_117", + "doc_38", + "doc_651", + "doc_148", + "doc_310", + "doc_73", + "doc_34", + "doc_56", + "doc_638", + "doc_174", + "doc_172", + "doc_623", + "doc_78", + "doc_183", + "doc_772", + "doc_660" + ] + } +} \ No newline at end of file diff --git a/benchmarks/ci/requirements.txt b/benchmarks/ci/requirements.txt new file mode 100644 index 00000000..1bd60f96 --- /dev/null +++ b/benchmarks/ci/requirements.txt @@ -0,0 +1,6 @@ +# CI benchmark harness — minimal dependencies +# NOTE: install from the repo root (pip resolves this path against the CWD): +# pip install -r benchmarks/ci/requirements.txt +./sdks/python/sdk +pytest>=7.0 +python-dotenv>=1.0.0 diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py new file mode 100644 index 00000000..1c63fbc9 --- /dev/null +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -0,0 +1,374 @@ +"""CI Benchmark Suite — Latency and Recall for Moss. + +Runs a fixed query set against a Moss index and records: + - p50 / p95 / p99 / mean latency (ms) + - recall@5 and recall@10 vs pre-computed ground truth + +Results are written to a JSON file (``--benchmark-output``) and optionally +compared against a checked-in baseline (``--baseline-file``) to catch +performance regressions. + +Usage:: + + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json +""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +import statistics +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from dotenv import load_dotenv + +load_dotenv() + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +CI_DIR = Path(__file__).resolve().parent + +# Subset of the full 100K corpus for CI speed. +DOC_COUNT = 1_000 +TOP_K_LATENCY = 5 +TOP_K_RECALL_5 = 5 +TOP_K_RECALL_10 = 10 +WARMUP_ROUNDS = 3 +QUERY_ROUNDS = 20 + +QUERIES = [ + "neural network training data", + "anomaly detection patterns", + "computer vision image processing", + "natural language processing", + "reinforcement learning rewards", + "transfer learning pretrained models", + "distributed computing systems", + "cryptographic data encryption", + "database indexing performance", + "knowledge graph entities", + "generative adversarial networks", + "attention mechanism transformers", + "dimensionality reduction compression", + "federated learning privacy", + "stream processing pipelines", +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _percentile(values: list[float], p: float) -> float: + """Compute the *p*-th percentile from a **sorted** list of values.""" + if not values: + return 0.0 + idx = max(int(math.ceil(p * len(values))) - 1, 0) + return values[idx] + + +_loop: asyncio.AbstractEventLoop | None = None + + +def _run(coro): + """Run *coro* on a single shared event loop. + + ``asyncio.get_event_loop()`` is deprecated (and raises on Python 3.14) + when no loop is running; ``asyncio.run()`` would create a fresh loop per + call, breaking clients that bind connections to the first loop. A single + explicit loop shared across the session avoids both problems. + """ + global _loop + if _loop is None: + _loop = asyncio.new_event_loop() + asyncio.set_event_loop(_loop) + return _loop.run_until_complete(coro) + + +def _git_sha() -> str: + """Return the short git SHA of HEAD, or 'unknown'.""" + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except Exception: + return "unknown" + + +# --------------------------------------------------------------------------- +# Session-scoped fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def moss_client(): + """Create a MossClient and load the benchmark index once per session.""" + # Import lazily — Moss native bindings may not be installed in every env. + from moss import MossClient, DocumentInfo, QueryOptions # noqa: F811 + + project_id = os.getenv("MOSS_PROJECT_ID") + project_key = os.getenv("MOSS_PROJECT_KEY") + if not project_id or not project_key: + pytest.skip("MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — skipping benchmarks") + + client = MossClient(project_id, project_key) + index_name = os.getenv("MOSS_INDEX_NAME", "benchmark-ci") + + async def _setup(): + # Create index with a small subset if it doesn't exist. + try: + await client.get_index(index_name) + except Exception: + # Load documents from the shared corpus file. + corpus_path = CI_DIR.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + pytest.skip(f"Corpus file not found: {corpus_path}") + with open(corpus_path) as f: + all_docs = json.load(f) + docs = [ + DocumentInfo( + id=d["id"], + text=d["text"], + metadata=d.get("metadata"), + ) + for d in all_docs[:DOC_COUNT] + ] + await client.create_index(index_name, docs, "moss-minilm") + + await client.load_index(index_name) + return client, index_name + + client, index_name = _run(_setup()) + yield client, index_name + + +@pytest.fixture(scope="session") +def ground_truth() -> dict[str, list[str]]: + """Load pre-computed ground truth document IDs per query.""" + gt_path = CI_DIR / "ground_truth.json" + if not gt_path.exists(): + pytest.skip(f"Ground truth file not found: {gt_path}") + with open(gt_path) as f: + data = json.load(f) + return data.get("queries", {}) + + +@pytest.fixture(scope="session") +def benchmark_results() -> dict: + """Mutable dict that accumulates results across tests in this session. + + The ``test_write_results`` finalizer serializes this to JSON. + """ + return { + "commit": _git_sha(), + "timestamp": datetime.now(timezone.utc).isoformat(), + "config": { + "doc_count": DOC_COUNT, + "query_rounds": QUERY_ROUNDS, + "warmup_rounds": WARMUP_ROUNDS, + "top_k_latency": TOP_K_LATENCY, + }, + "latency_ms": {}, + "recall": {}, + } + + +# --------------------------------------------------------------------------- +# Tests — run in declaration order via pytest-ordering or alphabetically +# --------------------------------------------------------------------------- + + +class TestBenchmarkLatency: + """Measure end-to-end query latency over multiple rounds.""" + + def test_latency(self, moss_client, benchmark_results): + from moss import QueryOptions + + client, index_name = moss_client + latencies: list[float] = [] + + async def _measure(): + # Warmup + for _ in range(WARMUP_ROUNDS): + for q in QUERIES: + await client.query( + index_name, q, QueryOptions(top_k=TOP_K_LATENCY, alpha=1) + ) + + # Measured rounds + for _ in range(QUERY_ROUNDS): + for q in QUERIES: + start = time.perf_counter() + await client.query( + index_name, q, QueryOptions(top_k=TOP_K_LATENCY, alpha=1) + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + latencies.append(elapsed_ms) + + _run(_measure()) + + latencies.sort() + result = { + "p50": round(_percentile(latencies, 0.50), 3), + "p95": round(_percentile(latencies, 0.95), 3), + "p99": round(_percentile(latencies, 0.99), 3), + "mean": round(statistics.mean(latencies), 3), + "stdev": round(statistics.stdev(latencies), 3) if len(latencies) >= 2 else 0.0, + "count": len(latencies), + } + benchmark_results["latency_ms"] = result + + # Print for CI logs + print(f"\n Latency ({len(latencies)} measurements):") + print(f" P50 : {result['p50']:.3f} ms") + print(f" P95 : {result['p95']:.3f} ms") + print(f" P99 : {result['p99']:.3f} ms") + print(f" Mean : {result['mean']:.3f} ms") + print(f" Stdev: {result['stdev']:.3f} ms") + + +class TestBenchmarkRecall: + """Measure recall@k against pre-computed ground truth.""" + + def test_recall(self, moss_client, ground_truth, benchmark_results): + from moss import QueryOptions + + client, index_name = moss_client + + recall_at_5_scores: list[float] = [] + recall_at_10_scores: list[float] = [] + + async def _evaluate(): + for q in QUERIES: + expected_ids = ground_truth.get(q, []) + if not expected_ids: + continue + + # recall@10 — fetch 10 results, also compute recall@5 + result = await client.query( + index_name, q, QueryOptions(top_k=TOP_K_RECALL_10, alpha=1) + ) + returned_ids = [doc.id for doc in result.docs] + + # recall@5 + expected_5 = set(expected_ids[:TOP_K_RECALL_5]) + returned_5 = set(returned_ids[:TOP_K_RECALL_5]) + if expected_5: + recall_at_5_scores.append( + len(expected_5 & returned_5) / len(expected_5) + ) + + # recall@10 + expected_10 = set(expected_ids[:TOP_K_RECALL_10]) + returned_10 = set(returned_ids[:TOP_K_RECALL_10]) + if expected_10: + recall_at_10_scores.append( + len(expected_10 & returned_10) / len(expected_10) + ) + + _run(_evaluate()) + + recall_5 = round(statistics.mean(recall_at_5_scores), 4) if recall_at_5_scores else 0.0 + recall_10 = round(statistics.mean(recall_at_10_scores), 4) if recall_at_10_scores else 0.0 + + benchmark_results["recall"] = { + "recall_at_5": recall_5, + "recall_at_10": recall_10, + "queries_evaluated": len(recall_at_5_scores), + } + + print(f"\n Recall ({len(recall_at_5_scores)} queries evaluated):") + print(f" Recall@5 : {recall_5:.4f}") + print(f" Recall@10 : {recall_10:.4f}") + + +class TestRegressionGuard: + """Compare current run against the checked-in baseline.""" + + def test_no_latency_regression(self, request, benchmark_results): + baseline_path = request.config.getoption("--baseline-file") + threshold = request.config.getoption("--latency-threshold") + + if not baseline_path or not Path(baseline_path).exists(): + pytest.skip("No baseline file provided — skipping regression check") + + with open(baseline_path) as f: + baseline = json.load(f) + + baseline_p95 = baseline.get("latency_ms", {}).get("p95") + current_p95 = benchmark_results.get("latency_ms", {}).get("p95") + + if baseline_p95 is None or current_p95 is None: + pytest.skip("Latency data not yet available — run latency test first") + + if baseline_p95 == 0: + pytest.skip("Baseline p95 is zero — cannot compute regression ratio") + + regression = (current_p95 - baseline_p95) / baseline_p95 + + print(f"\n Latency regression check:") + print(f" Baseline P95 : {baseline_p95:.3f} ms") + print(f" Current P95 : {current_p95:.3f} ms") + print(f" Change : {regression:+.1%}") + print(f" Threshold : {threshold:.0%}") + + assert regression <= threshold, ( + f"P95 latency regressed by {regression:+.1%} " + f"(baseline={baseline_p95:.3f}ms, current={current_p95:.3f}ms, " + f"threshold={threshold:.0%})" + ) + + def test_no_recall_regression(self, request, benchmark_results): + baseline_path = request.config.getoption("--baseline-file") + threshold = request.config.getoption("--recall-threshold") + + if not baseline_path or not Path(baseline_path).exists(): + pytest.skip("No baseline file provided — skipping regression check") + + with open(baseline_path) as f: + baseline = json.load(f) + + baseline_recall = baseline.get("recall", {}).get("recall_at_5") + current_recall = benchmark_results.get("recall", {}).get("recall_at_5") + + if baseline_recall is None or current_recall is None: + pytest.skip("Recall data not yet available — run recall test first") + + drop = baseline_recall - current_recall + + print(f"\n Recall regression check:") + print(f" Baseline Recall@5 : {baseline_recall:.4f}") + print(f" Current Recall@5 : {current_recall:.4f}") + print(f" Drop : {drop:+.4f}") + print(f" Threshold : {threshold:.4f}") + + assert drop <= threshold, ( + f"Recall@5 dropped by {drop:.4f} " + f"(baseline={baseline_recall:.4f}, current={current_recall:.4f}, " + f"threshold={threshold:.4f})" + ) + + +class TestWriteResults: + """Serialize benchmark results to JSON (always runs last).""" + + def test_write_results(self, request, benchmark_results): + output_path = request.config.getoption("--benchmark-output") + with open(output_path, "w") as f: + json.dump(benchmark_results, f, indent=2) + print(f"\n Results written to: {output_path}") From ded57ea7cdf770d2c591ae6461c7060939b7f6f3 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Mon, 20 Jul 2026 11:37:14 +0530 Subject: [PATCH 04/17] Address review feedback: harden generator, workflow, and docs - Share QUERIES/index config between tests and generator via bench_queries.py so the sets can never drift - Check index existence via list_indexes() instead of a broad except around get_index, so auth/network errors surface rather than silently triggering index creation - Add --recreate flag to generate_ground_truth.py to rebuild the index after corpus/model changes before capturing ground truth - Run baseline-update workflow dispatches without the regression comparison so an intentional perf change can't fail its own baseline refresh; clarify that persisting requires committing the artifact - Activate the recall guard: baseline now carries measured recall (hardware-independent); latency stays placeholder until a CI- runner baseline is committed - Pin benchmark dependencies exactly to keep results attributable to repo changes - Docs: fix non-runnable shell example, qualify CI coverage (fork PRs skip), document ground truth as a ranking-stability reference Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 24 ++++-- benchmarks/README.md | 9 ++- benchmarks/ci/README.md | 27 +++++-- benchmarks/ci/baseline.json | 12 +-- benchmarks/ci/bench_queries.py | 28 +++++++ benchmarks/ci/generate_ground_truth.py | 103 ++++++++++++++----------- benchmarks/ci/requirements.txt | 6 +- benchmarks/ci/test_bench_ci_moss.py | 42 ++++------ 8 files changed, 157 insertions(+), 94 deletions(-) create mode 100644 benchmarks/ci/bench_queries.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c6217009..8fa62c83 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,11 +36,20 @@ jobs: MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} run: | - pytest benchmarks/ci/ -v \ - --benchmark-output=benchmark_results.json \ - --baseline-file=benchmarks/ci/baseline.json \ - --latency-threshold=0.20 \ - --recall-threshold=0.05 + # Baseline-update runs skip the regression comparison: comparing + # against the baseline being replaced would fail the run (and skip + # the copy step) exactly when an intentional change moved the numbers. + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && \ + [ "${{ github.event.inputs.update_baseline }}" = "true" ]; then + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json + else + pytest benchmarks/ci/ -v \ + --benchmark-output=benchmark_results.json \ + --baseline-file=benchmarks/ci/baseline.json \ + --latency-threshold=0.20 \ + --recall-threshold=0.05 + fi - name: Upload results artifact uses: actions/upload-artifact@v4 @@ -57,7 +66,8 @@ jobs: run: | echo "Copying benchmark_results.json → benchmarks/ci/baseline.json" cp benchmark_results.json benchmarks/ci/baseline.json - echo "Updated baseline:" + echo "New baseline (runner copy only — NOT committed):" cat benchmarks/ci/baseline.json echo "" - echo "NOTE: To persist this change, commit and push benchmarks/ci/baseline.json" + echo "To persist: download the benchmark-results-${{ github.sha }} artifact," + echo "copy it to benchmarks/ci/baseline.json, and commit." diff --git a/benchmarks/README.md b/benchmarks/README.md index 1831c3e2..b089ec16 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -82,8 +82,11 @@ EMBEDDING_DIMENSION=768 ## CI Benchmark Suite For automated regression testing in CI, see [`benchmarks/ci/`](ci/). -This suite runs on every push to `main` and on PRs, tracking p50/p95/p99 -latency and recall@k per commit. It compares against a checked-in baseline -and fails the build if regressions exceed configurable thresholds. +The workflow runs on every push to `main` and on PRs; benchmark tests skip +when Moss credentials are unavailable (such as fork PRs). Authenticated runs +track p50/p95/p99 latency and recall@k per commit and compare against a +checked-in baseline, failing the build when regressions exceed the +configured thresholds. The latency guard activates once a baseline measured +on CI runners replaces the initial placeholder. See [`benchmarks/ci/README.md`](ci/README.md) for full documentation. \ No newline at end of file diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index d04a90e2..d874757e 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -47,6 +47,14 @@ python benchmarks/ci/generate_ground_truth.py This queries Moss with `top_k=50` for each benchmark query and writes the expected document IDs to `ground_truth.json`. Commit the updated file. +After a corpus or model change, pass `--recreate` so the index is rebuilt +from the current corpus before the ground truth is captured. + +> **Note**: the ground truth is a *ranking-stability reference* generated by +> Moss itself at a known-good commit — not an independent relevance judgment. +> The recall gate detects changes in retrieval behavior; an intentional +> relevance improvement will trip it and should be accompanied by a +> regenerated ground truth in the same PR. ## How regression detection works @@ -58,9 +66,11 @@ The harness compares the current run's metrics against `baseline.json`: Thresholds are configurable via CLI flags: ```bash +# --latency-threshold: max fractional P95 increase (0.15 = 15%) +# --recall-threshold: max absolute recall@5 drop (0.03 = 3pp) pytest benchmarks/ci/ -v \ - --latency-threshold=0.15 \ # 15% max P95 regression - --recall-threshold=0.03 # 3pp max recall drop + --latency-threshold=0.15 \ + --recall-threshold=0.03 ``` ## Updating the baseline @@ -68,10 +78,14 @@ pytest benchmarks/ci/ -v \ After a legitimate performance change (e.g., model upgrade, index config change), update the baseline: -1. **Via GitHub Actions**: Trigger the `Benchmark` workflow manually with - `update_baseline=true` -2. **Manually**: Copy a CI run's `benchmark_results.json` artifact to - `benchmarks/ci/baseline.json` and commit +1. Trigger the `Benchmark` workflow manually with `update_baseline=true`. + This runs the suite **without** the regression comparison (so an + intentional change can't fail its own baseline run) and produces a fresh + `benchmark-results-` artifact. It does **not** commit anything — + the runner copy is discarded when the job ends. +2. Download that artifact, copy it to `benchmarks/ci/baseline.json`, and + commit. Baselines should always come from CI runners — latency measured + on other hardware is not comparable. ## CI integration @@ -84,6 +98,7 @@ available for download from the Actions tab. | File | Purpose | |------|---------| | `test_bench_ci_moss.py` | Main test module (latency, recall, regression guard) | +| `bench_queries.py` | Shared query set + index config (used by tests and generator) | | `conftest.py` | Pytest CLI flags | | `generate_ground_truth.py` | One-time ground truth generator | | `ground_truth.json` | Pre-computed expected results per query | diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json index 4471b2f8..4d645064 100644 --- a/benchmarks/ci/baseline.json +++ b/benchmarks/ci/baseline.json @@ -1,7 +1,7 @@ { - "commit": "initial", - "timestamp": "2025-07-19T00:00:00+00:00", - "_note": "Placeholder baseline — update via 'workflow_dispatch' or copy a CI artifact.", + "commit": "f4732a1", + "timestamp": "2026-07-20T05:35:00+00:00", + "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero — latency is hardware-dependent, so the guard skips until a baseline captured on CI runners replaces this file (run the Benchmark workflow, download the benchmark-results- artifact, commit it here).", "latency_ms": { "p50": 0, "p95": 0, @@ -11,9 +11,9 @@ "count": 0 }, "recall": { - "recall_at_5": 0, - "recall_at_10": 0, - "queries_evaluated": 0 + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "queries_evaluated": 15 }, "config": { "doc_count": 1000, diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py new file mode 100644 index 00000000..094fc58d --- /dev/null +++ b/benchmarks/ci/bench_queries.py @@ -0,0 +1,28 @@ +"""Shared configuration for the CI benchmark suite. + +Both the benchmark tests (``test_bench_ci_moss.py``) and the ground-truth +generator (``generate_ground_truth.py``) import from this module so the +query set can never drift between generation and evaluation. +""" + +INDEX_NAME_DEFAULT = "benchmark-ci" +MODEL_ID = "moss-minilm" +DOC_COUNT = 1_000 + +QUERIES = [ + "neural network training data", + "anomaly detection patterns", + "computer vision image processing", + "natural language processing", + "reinforcement learning rewards", + "transfer learning pretrained models", + "distributed computing systems", + "cryptographic data encryption", + "database indexing performance", + "knowledge graph entities", + "generative adversarial networks", + "attention mechanism transformers", + "dimensionality reduction compression", + "federated learning privacy", + "stream processing pipelines", +] diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py index 817f3986..72b31f79 100644 --- a/benchmarks/ci/generate_ground_truth.py +++ b/benchmarks/ci/generate_ground_truth.py @@ -2,19 +2,30 @@ """Generate ground truth for CI benchmark recall computation. Queries the Moss index with a large top_k and records the returned document -IDs as the "expected" relevant set for each benchmark query. Run this once -(or whenever the index/model changes) and commit the output. +IDs as the "expected" set for each benchmark query. Run this once (or +whenever the index/model changes) and commit the output. + +.. note:: + This is a **ranking-stability reference**, not an independent relevance + judgment: the expected IDs come from Moss itself at a known-good commit. + The recall gate therefore detects *changes in retrieval behavior* (the + goal of a regression guard), and will also flag intentional relevance + improvements — regenerate and commit a new ground truth in that case. Usage:: # Ensure MOSS_PROJECT_ID and MOSS_PROJECT_KEY are set python benchmarks/ci/generate_ground_truth.py + # After a corpus or model change, rebuild the index first: + python benchmarks/ci/generate_ground_truth.py --recreate + Output is written to ``benchmarks/ci/ground_truth.json``. """ from __future__ import annotations +import argparse import asyncio import json import os @@ -23,38 +34,38 @@ from dotenv import load_dotenv -load_dotenv() +from bench_queries import DOC_COUNT, INDEX_NAME_DEFAULT, MODEL_ID, QUERIES -# Re-use the same query set as the CI benchmark. -QUERIES = [ - "neural network training data", - "anomaly detection patterns", - "computer vision image processing", - "natural language processing", - "reinforcement learning rewards", - "transfer learning pretrained models", - "distributed computing systems", - "cryptographic data encryption", - "database indexing performance", - "knowledge graph entities", - "generative adversarial networks", - "attention mechanism transformers", - "dimensionality reduction compression", - "federated learning privacy", - "stream processing pipelines", -] +load_dotenv() # Fetch a generous top_k so recall@5 and recall@10 can be evaluated # against a superset of relevant results. GROUND_TRUTH_TOP_K = 50 -async def main() -> None: - from moss import MossClient, DocumentInfo, QueryOptions +async def _create_index(client, index_name: str) -> None: + from moss import DocumentInfo + + corpus_path = Path(__file__).resolve().parent.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + print(f"Error: Corpus file not found: {corpus_path}") + sys.exit(1) + with open(corpus_path) as f: + all_docs = json.load(f) + docs = [ + DocumentInfo(id=d["id"], text=d["text"], metadata=d.get("metadata")) + for d in all_docs[:DOC_COUNT] + ] + result = await client.create_index(index_name, docs, MODEL_ID) + print(f"Created index '{index_name}' with {result.doc_count} docs") + + +async def main(recreate: bool) -> None: + from moss import MossClient, QueryOptions project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") - index_name = os.getenv("MOSS_INDEX_NAME", "benchmark-ci") + index_name = os.getenv("MOSS_INDEX_NAME", INDEX_NAME_DEFAULT) if not project_id or not project_key: print("Error: MOSS_PROJECT_ID and MOSS_PROJECT_KEY must be set.") @@ -62,23 +73,20 @@ async def main() -> None: client = MossClient(project_id, project_key) - # Ensure the index exists (create with a 1K subset if needed). - try: - await client.get_index(index_name) + # Determine existence explicitly (rather than treating any get_index + # failure as "missing") so auth/network errors surface instead of + # silently triggering index creation. + existing = {idx.name for idx in await client.list_indexes()} + + if index_name in existing and recreate: + print(f"--recreate: deleting existing index '{index_name}'") + await client.delete_index(index_name) + existing.discard(index_name) + + if index_name in existing: print(f"Using existing index '{index_name}'") - except Exception: - corpus_path = Path(__file__).resolve().parent.parent / "bench_100k_docs.json" - if not corpus_path.exists(): - print(f"Error: Corpus file not found: {corpus_path}") - sys.exit(1) - with open(corpus_path) as f: - all_docs = json.load(f) - docs = [ - DocumentInfo(id=d["id"], text=d["text"], metadata=d.get("metadata")) - for d in all_docs[:1000] - ] - result = await client.create_index(index_name, docs, "moss-minilm") - print(f"Created index '{index_name}' with {result.doc_count} docs") + else: + await _create_index(client, index_name) await client.load_index(index_name) @@ -95,10 +103,10 @@ async def main() -> None: print(f" '{q}' → {len(doc_ids)} results") output = { - "model": "moss-minilm", + "model": MODEL_ID, "top_k": GROUND_TRUTH_TOP_K, "index_name": index_name, - "doc_count": 1000, + "doc_count": DOC_COUNT, "queries": ground_truth, } @@ -111,4 +119,13 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--recreate", + action="store_true", + help="Delete and rebuild the benchmark index from the corpus before " + "querying. Required after a corpus or embedding-model change so the " + "ground truth reflects the current data.", + ) + args = parser.parse_args() + asyncio.run(main(recreate=args.recreate)) diff --git a/benchmarks/ci/requirements.txt b/benchmarks/ci/requirements.txt index 1bd60f96..60474cae 100644 --- a/benchmarks/ci/requirements.txt +++ b/benchmarks/ci/requirements.txt @@ -1,6 +1,8 @@ # CI benchmark harness — minimal dependencies # NOTE: install from the repo root (pip resolves this path against the CWD): # pip install -r benchmarks/ci/requirements.txt +# Versions are pinned exactly so benchmark numbers stay attributable to +# repository changes rather than dependency drift; bump deliberately. ./sdks/python/sdk -pytest>=7.0 -python-dotenv>=1.0.0 +pytest==9.1.1 +python-dotenv==1.2.2 diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 1c63fbc9..5a096df6 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -30,6 +30,8 @@ import pytest from dotenv import load_dotenv +from bench_queries import DOC_COUNT, INDEX_NAME_DEFAULT, MODEL_ID, QUERIES + load_dotenv() # --------------------------------------------------------------------------- @@ -38,32 +40,14 @@ CI_DIR = Path(__file__).resolve().parent -# Subset of the full 100K corpus for CI speed. -DOC_COUNT = 1_000 +# DOC_COUNT (1K subset of the full 100K corpus, for CI speed), the query set, +# and the model id are shared with generate_ground_truth.py via bench_queries. TOP_K_LATENCY = 5 TOP_K_RECALL_5 = 5 TOP_K_RECALL_10 = 10 WARMUP_ROUNDS = 3 QUERY_ROUNDS = 20 -QUERIES = [ - "neural network training data", - "anomaly detection patterns", - "computer vision image processing", - "natural language processing", - "reinforcement learning rewards", - "transfer learning pretrained models", - "distributed computing systems", - "cryptographic data encryption", - "database indexing performance", - "knowledge graph entities", - "generative adversarial networks", - "attention mechanism transformers", - "dimensionality reduction compression", - "federated learning privacy", - "stream processing pipelines", -] - # --------------------------------------------------------------------------- # Helpers @@ -127,13 +111,14 @@ def moss_client(): pytest.skip("MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — skipping benchmarks") client = MossClient(project_id, project_key) - index_name = os.getenv("MOSS_INDEX_NAME", "benchmark-ci") + index_name = os.getenv("MOSS_INDEX_NAME", INDEX_NAME_DEFAULT) async def _setup(): - # Create index with a small subset if it doesn't exist. - try: - await client.get_index(index_name) - except Exception: + # Determine existence explicitly (rather than treating any get_index + # failure as "missing") so auth/network errors surface instead of + # silently triggering index creation. + existing = {idx.name for idx in await client.list_indexes()} + if index_name not in existing: # Load documents from the shared corpus file. corpus_path = CI_DIR.parent / "bench_100k_docs.json" if not corpus_path.exists(): @@ -148,7 +133,7 @@ async def _setup(): ) for d in all_docs[:DOC_COUNT] ] - await client.create_index(index_name, docs, "moss-minilm") + await client.create_index(index_name, docs, MODEL_ID) await client.load_index(index_name) return client, index_name @@ -189,7 +174,10 @@ def benchmark_results() -> dict: # --------------------------------------------------------------------------- -# Tests — run in declaration order via pytest-ordering or alphabetically +# Tests — pytest collects these in declaration order (measure → guard → +# write). The ordering is a soft dependency only: the guard and writer +# degrade gracefully (skip / write partial results) if measurement data is +# missing, so a random-ordering plugin breaks nothing, it just skips checks. # --------------------------------------------------------------------------- From 196d2f86f8e0aa29a3ff3194751f0fa2f466b1d2 Mon Sep 17 00:00:00 2001 From: Sravan Avvaru <81159574+Sravan1011@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:39:11 +0530 Subject: [PATCH 05/17] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- benchmarks/ci/test_bench_ci_moss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 5a096df6..86d4dd55 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -103,7 +103,7 @@ def _git_sha() -> str: def moss_client(): """Create a MossClient and load the benchmark index once per session.""" # Import lazily — Moss native bindings may not be installed in every env. - from moss import MossClient, DocumentInfo, QueryOptions # noqa: F811 +from moss import MossClient, DocumentInfo project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") From 26ff7d474c35f4bd27ce4b89154878182526f1f4 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Wed, 22 Jul 2026 09:55:26 +0530 Subject: [PATCH 06/17] Fix broken indent from suggested change; address Copilot findings - Restore indentation of the lazy moss import inside the moss_client fixture (auto-applied suggestion dedented it, breaking collection) - Fail fast when a benchmark query has no ground-truth entry instead of silently shrinking the evaluated set and inflating recall - Create parent directories for --benchmark-output before writing - Correct --recall-threshold help text (guard compares recall@5) Co-Authored-By: Claude Fable 5 --- benchmarks/ci/conftest.py | 2 +- benchmarks/ci/test_bench_ci_moss.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/benchmarks/ci/conftest.py b/benchmarks/ci/conftest.py index 3d0e1f1a..cd4a7342 100644 --- a/benchmarks/ci/conftest.py +++ b/benchmarks/ci/conftest.py @@ -31,6 +31,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--recall-threshold", type=float, default=0.05, - help="Max allowed absolute decrease in recall@k vs baseline " + help="Max allowed absolute decrease in recall@5 vs baseline " "(default: 0.05 = 5 percentage points)", ) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 86d4dd55..d496f1d6 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -103,7 +103,7 @@ def _git_sha() -> str: def moss_client(): """Create a MossClient and load the benchmark index once per session.""" # Import lazily — Moss native bindings may not be installed in every env. -from moss import MossClient, DocumentInfo + from moss import MossClient, DocumentInfo project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") @@ -243,9 +243,14 @@ def test_recall(self, moss_client, ground_truth, benchmark_results): async def _evaluate(): for q in QUERIES: - expected_ids = ground_truth.get(q, []) + expected_ids = ground_truth.get(q) if not expected_ids: - continue + # Silently skipping would shrink the evaluated set and + # inflate recall — fail loudly instead. + raise AssertionError( + f"Ground truth missing results for query {q!r}; " + "regenerate benchmarks/ci/ground_truth.json" + ) # recall@10 — fetch 10 results, also compute recall@5 result = await client.query( @@ -356,7 +361,8 @@ class TestWriteResults: """Serialize benchmark results to JSON (always runs last).""" def test_write_results(self, request, benchmark_results): - output_path = request.config.getoption("--benchmark-output") + output_path = Path(request.config.getoption("--benchmark-output")) + output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: json.dump(benchmark_results, f, indent=2) print(f"\n Results written to: {output_path}") From 16313b11422a8c363426fd7ad00e9a46dd86c817 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Wed, 22 Jul 2026 10:02:22 +0530 Subject: [PATCH 07/17] Fix ruff F541: remove f-prefix from placeholder-free strings Co-Authored-By: Claude Fable 5 --- benchmarks/ci/test_bench_ci_moss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index d496f1d6..81a35bbb 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -314,7 +314,7 @@ def test_no_latency_regression(self, request, benchmark_results): regression = (current_p95 - baseline_p95) / baseline_p95 - print(f"\n Latency regression check:") + print("\n Latency regression check:") print(f" Baseline P95 : {baseline_p95:.3f} ms") print(f" Current P95 : {current_p95:.3f} ms") print(f" Change : {regression:+.1%}") @@ -344,7 +344,7 @@ def test_no_recall_regression(self, request, benchmark_results): drop = baseline_recall - current_recall - print(f"\n Recall regression check:") + print("\n Recall regression check:") print(f" Baseline Recall@5 : {baseline_recall:.4f}") print(f" Current Recall@5 : {current_recall:.4f}") print(f" Drop : {drop:+.4f}") From ffd185b6b82cde0248584667db873100a21277c7 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 08:45:02 +0530 Subject: [PATCH 08/17] fix: fail trusted CI on missing secrets; signature-derived benchmark index Addresses two blocking review findings: 1. Missing credentials previously skipped unconditionally, so a missing or misconfigured secret on push/same-repo PRs turned the workflow into a green no-op. The workflow now exports ALLOW_BENCHMARK_SKIP=1 only for fork PRs (which cannot read repository secrets); in any other CI run, absent credentials pytest.fail() with an actionable message. Local runs (no CI env) still skip. The credential gate also moved above the moss import so fork PRs skip cleanly even where native bindings are unavailable. 2. The fixed benchmark-ci index name could silently reuse remote state built from a different corpus, DOC_COUNT, or model, benchmarking stale data. The index name is now derived from a content signature (model id + DOC_COUNT + the exact corpus slice, sha256-truncated): changed inputs produce a different name and the index is rebuilt. generate_ground_truth.py derives the same name and embeds the signature in ground_truth.json; the recall fixture fails with a regenerate hint when the stored signature does not match the current inputs. The signature and index name are also recorded in the results artifact for traceability. MOSS_INDEX_NAME remains an explicit override. Verified all three credential paths locally: local no-creds skips, CI=true no-creds fails with the configured message, and ALLOW_BENCHMARK_SKIP=1 skips. ground_truth.json restamped with the computed signature (73d5e83176f7). Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 5 ++ benchmarks/ci/README.md | 21 +++++++ benchmarks/ci/bench_queries.py | 40 ++++++++++++- benchmarks/ci/generate_ground_truth.py | 35 ++++++++--- benchmarks/ci/ground_truth.json | 3 +- benchmarks/ci/test_bench_ci_moss.py | 82 +++++++++++++++++++++----- 6 files changed, 158 insertions(+), 28 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8fa62c83..8a3a6bcd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -35,6 +35,11 @@ jobs: env: MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} + # Fork PRs cannot read repository secrets, so missing credentials + # are expected there and the suite may skip. On trusted runs + # (push to main, same-repo PRs, manual dispatch) missing secrets + # make the suite FAIL instead of passing as a green no-op. + ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && '1' || '0' }} run: | # Baseline-update runs skip the regression comparison: comparing # against the baseline being replaced would fail the run (and skip diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index d874757e..5a55570f 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -87,6 +87,27 @@ change), update the baseline: commit. Baselines should always come from CI runners — latency measured on other hardware is not comparable. +## Index naming and staleness protection + +The benchmark index name is derived from a content signature — +`benchmark-ci-` where the hash covers the model id, `DOC_COUNT`, and +the exact corpus slice being indexed. If any of those inputs change, the +name changes and the index is rebuilt from the current corpus, so a stale +remote index can never be silently benchmarked. `ground_truth.json` embeds +the same signature; the recall test **fails** (with a regenerate hint) when +the ground truth was generated from different inputs. Set `MOSS_INDEX_NAME` +to override the derived name (this bypasses the staleness protection for +the index itself; the ground-truth signature check still applies). + +## Credentials policy + +- **Trusted CI runs** (push to `main`, same-repo PRs, manual dispatch): + missing `MOSS_PROJECT_ID` / `MOSS_PROJECT_KEY` **fails** the suite — a + misconfigured secret must not turn the workflow into a green no-op. +- **Fork PRs**: secrets are not available to forks, so the workflow sets + `ALLOW_BENCHMARK_SKIP=1` and the suite skips cleanly. +- **Local runs** (no `CI` env var): missing credentials skip the suite. + ## CI integration The benchmark runs as a GitHub Actions job (`.github/workflows/benchmark.yml`). diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py index 094fc58d..e50a0a09 100644 --- a/benchmarks/ci/bench_queries.py +++ b/benchmarks/ci/bench_queries.py @@ -2,13 +2,49 @@ Both the benchmark tests (``test_bench_ci_moss.py``) and the ground-truth generator (``generate_ground_truth.py``) import from this module so the -query set can never drift between generation and evaluation. +query set, index naming, and corpus signature can never drift between +generation and evaluation. """ -INDEX_NAME_DEFAULT = "benchmark-ci" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +INDEX_NAME_PREFIX = "benchmark-ci" MODEL_ID = "moss-minilm" DOC_COUNT = 1_000 + +def load_corpus_slice(corpus_path: Path) -> list[dict[str, Any]]: + """Load the first ``DOC_COUNT`` documents of the shared corpus file.""" + with open(corpus_path) as f: + all_docs = json.load(f) + return all_docs[:DOC_COUNT] + + +def corpus_signature(docs: list[dict[str, Any]]) -> str: + """Short content hash of everything the benchmark index is built from. + + Covers the model id, DOC_COUNT, and the exact corpus slice that gets + indexed. Any change to those inputs yields a different signature — and + therefore a different index name via ``index_name_for`` — so a stale + remote index can never be silently reused against mismatched data. + """ + h = hashlib.sha256() + h.update(MODEL_ID.encode()) + h.update(str(DOC_COUNT).encode()) + for d in docs: + h.update(json.dumps(d, sort_keys=True).encode()) + return h.hexdigest()[:12] + + +def index_name_for(signature: str) -> str: + """Benchmark index name derived from the corpus/model signature.""" + return f"{INDEX_NAME_PREFIX}-{signature}" + QUERIES = [ "neural network training data", "anomaly detection patterns", diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py index 72b31f79..a2737a5b 100644 --- a/benchmarks/ci/generate_ground_truth.py +++ b/benchmarks/ci/generate_ground_truth.py @@ -34,7 +34,14 @@ from dotenv import load_dotenv -from bench_queries import DOC_COUNT, INDEX_NAME_DEFAULT, MODEL_ID, QUERIES +from bench_queries import ( + DOC_COUNT, + MODEL_ID, + QUERIES, + corpus_signature, + index_name_for, + load_corpus_slice, +) load_dotenv() @@ -43,18 +50,20 @@ GROUND_TRUTH_TOP_K = 50 -async def _create_index(client, index_name: str) -> None: - from moss import DocumentInfo - +def _corpus_slice() -> list[dict]: corpus_path = Path(__file__).resolve().parent.parent / "bench_100k_docs.json" if not corpus_path.exists(): print(f"Error: Corpus file not found: {corpus_path}") sys.exit(1) - with open(corpus_path) as f: - all_docs = json.load(f) + return load_corpus_slice(corpus_path) + + +async def _create_index(client, index_name: str, corpus_slice: list[dict]) -> None: + from moss import DocumentInfo + docs = [ DocumentInfo(id=d["id"], text=d["text"], metadata=d.get("metadata")) - for d in all_docs[:DOC_COUNT] + for d in corpus_slice ] result = await client.create_index(index_name, docs, MODEL_ID) print(f"Created index '{index_name}' with {result.doc_count} docs") @@ -65,7 +74,12 @@ async def main(recreate: bool) -> None: project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") - index_name = os.getenv("MOSS_INDEX_NAME", INDEX_NAME_DEFAULT) + # Same derivation as the benchmark tests: the index name embeds the + # corpus/model signature, so generation and evaluation can never target + # indexes built from different inputs. + corpus_slice = _corpus_slice() + signature = corpus_signature(corpus_slice) + index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(signature) if not project_id or not project_key: print("Error: MOSS_PROJECT_ID and MOSS_PROJECT_KEY must be set.") @@ -86,7 +100,7 @@ async def main(recreate: bool) -> None: if index_name in existing: print(f"Using existing index '{index_name}'") else: - await _create_index(client, index_name) + await _create_index(client, index_name, corpus_slice) await client.load_index(index_name) @@ -107,6 +121,9 @@ async def main(recreate: bool) -> None: "top_k": GROUND_TRUTH_TOP_K, "index_name": index_name, "doc_count": DOC_COUNT, + # Validated by the benchmark tests: recall is only evaluated when the + # current corpus/model signature matches the one used at generation. + "signature": signature, "queries": ground_truth, } diff --git a/benchmarks/ci/ground_truth.json b/benchmarks/ci/ground_truth.json index 76d2e9a9..641da94f 100644 --- a/benchmarks/ci/ground_truth.json +++ b/benchmarks/ci/ground_truth.json @@ -1,8 +1,9 @@ { "model": "moss-minilm", "top_k": 50, - "index_name": "benchmark-ci", + "index_name": "benchmark-ci-73d5e83176f7", "doc_count": 1000, + "signature": "73d5e83176f7", "queries": { "neural network training data": [ "doc_113", diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 81a35bbb..0954a02b 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -30,7 +30,14 @@ import pytest from dotenv import load_dotenv -from bench_queries import DOC_COUNT, INDEX_NAME_DEFAULT, MODEL_ID, QUERIES +from bench_queries import ( + DOC_COUNT, + MODEL_ID, + QUERIES, + corpus_signature, + index_name_for, + load_corpus_slice, +) load_dotenv() @@ -100,18 +107,52 @@ def _git_sha() -> str: @pytest.fixture(scope="session") -def moss_client(): - """Create a MossClient and load the benchmark index once per session.""" - # Import lazily — Moss native bindings may not be installed in every env. - from moss import MossClient, DocumentInfo +def corpus_slice() -> list[dict]: + """The exact corpus slice the benchmark index is built from.""" + corpus_path = CI_DIR.parent / "bench_100k_docs.json" + if not corpus_path.exists(): + pytest.skip(f"Corpus file not found: {corpus_path}") + return load_corpus_slice(corpus_path) + + +@pytest.fixture(scope="session") +def corpus_sig(corpus_slice) -> str: + """Content signature of model + DOC_COUNT + corpus slice.""" + return corpus_signature(corpus_slice) + + +@pytest.fixture(scope="session") +def moss_client(corpus_slice, corpus_sig): + """Create a MossClient and load the benchmark index once per session. + The index name embeds ``corpus_sig``, so an index built from a different + corpus, DOC_COUNT, or model can never be silently reused — mismatched + inputs produce a different name and the index is (re)created from the + current corpus. + """ project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") if not project_id or not project_key: + if os.getenv("ALLOW_BENCHMARK_SKIP") == "1": + pytest.skip( + "MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — fork PR without " + "secrets, skipping benchmarks" + ) + if os.getenv("CI"): + pytest.fail( + "MOSS_PROJECT_ID / MOSS_PROJECT_KEY are not set in a trusted CI " + "run — the benchmark workflow would otherwise pass as a green " + "no-op. Configure the repository secrets, or export " + "ALLOW_BENCHMARK_SKIP=1 for runs that legitimately lack them." + ) pytest.skip("MOSS_PROJECT_ID / MOSS_PROJECT_KEY not set — skipping benchmarks") + # Import lazily (and only once credentials are known to exist) — Moss + # native bindings may not be installed in every env. + from moss import MossClient, DocumentInfo + client = MossClient(project_id, project_key) - index_name = os.getenv("MOSS_INDEX_NAME", INDEX_NAME_DEFAULT) + index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig) async def _setup(): # Determine existence explicitly (rather than treating any get_index @@ -119,19 +160,13 @@ async def _setup(): # silently triggering index creation. existing = {idx.name for idx in await client.list_indexes()} if index_name not in existing: - # Load documents from the shared corpus file. - corpus_path = CI_DIR.parent / "bench_100k_docs.json" - if not corpus_path.exists(): - pytest.skip(f"Corpus file not found: {corpus_path}") - with open(corpus_path) as f: - all_docs = json.load(f) docs = [ DocumentInfo( id=d["id"], text=d["text"], metadata=d.get("metadata"), ) - for d in all_docs[:DOC_COUNT] + for d in corpus_slice ] await client.create_index(index_name, docs, MODEL_ID) @@ -143,18 +178,31 @@ async def _setup(): @pytest.fixture(scope="session") -def ground_truth() -> dict[str, list[str]]: - """Load pre-computed ground truth document IDs per query.""" +def ground_truth(corpus_sig) -> dict[str, list[str]]: + """Load pre-computed ground truth document IDs per query. + + Fails (not skips) when the ground truth was generated from a different + corpus/model/DOC_COUNT — evaluating recall against mismatched expected + ids would mask corpus or model regressions. + """ gt_path = CI_DIR / "ground_truth.json" if not gt_path.exists(): pytest.skip(f"Ground truth file not found: {gt_path}") with open(gt_path) as f: data = json.load(f) + gt_sig = data.get("signature") + if gt_sig != corpus_sig: + pytest.fail( + f"ground_truth.json signature {gt_sig!r} does not match the current " + f"corpus/model signature {corpus_sig!r} — the corpus, DOC_COUNT, or " + "model changed since generation. Regenerate with: " + "python benchmarks/ci/generate_ground_truth.py --recreate" + ) return data.get("queries", {}) @pytest.fixture(scope="session") -def benchmark_results() -> dict: +def benchmark_results(corpus_sig) -> dict: """Mutable dict that accumulates results across tests in this session. The ``test_write_results`` finalizer serializes this to JSON. @@ -167,6 +215,8 @@ def benchmark_results() -> dict: "query_rounds": QUERY_ROUNDS, "warmup_rounds": WARMUP_ROUNDS, "top_k_latency": TOP_K_LATENCY, + "signature": corpus_sig, + "index_name": os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig), }, "latency_ms": {}, "recall": {}, From 7df6fd3fb1769a58656adf8a8e354dd3148cf59b Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:02:57 +0530 Subject: [PATCH 09/17] fix: fail on missing required inputs; serialize runs; validate baseline compatibility Addresses three review findings: - BLOCKING: missing required inputs (corpus file, ground_truth.json) used the same unconditional pytest.skip as optional paths, so trusted CI could go green with the whole suite skipped. A shared _missing_required_input() helper now applies the credentials policy everywhere: skip on fork PRs (ALLOW_BENCHMARK_SKIP=1) and local runs, fail in trusted CI. An explicitly provided --baseline-file that does not exist now fails outright in every environment: asking for a regression comparison and silently not getting one is a config error. - CONSIDER (concurrency): all benchmark jobs share one Moss project and the deterministic benchmark-ci- index, so parallel PR/push runs could race on first index creation and contaminate each other's P95. The workflow now uses a global concurrency group with cancel-in-progress: false so runs queue instead of overlapping. - CONSIDER (baseline/query-set compatibility): the signature only covered corpus/model inputs. bench_queries now exposes query_set_hash(); ground_truth.json embeds it (recall fails with a regenerate hint when QUERIES drifted), benchmark results record it in config, and both regression guards fail via _assert_baseline_compatible() unless the baseline's config matches the current run on signature, query-set hash, doc count, rounds, and top_k. baseline.json and ground_truth.json restamped accordingly; index_name is deliberately excluded from the comparison since MOSS_INDEX_NAME may override it without changing what is measured. Verified locally: local no-creds skips, CI=true no-creds fails, fork-PR skips, and a provided-but-missing baseline file fails both guards. Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 8 +++ benchmarks/ci/README.md | 21 ++++++-- benchmarks/ci/baseline.json | 9 ++-- benchmarks/ci/bench_queries.py | 14 +++++ benchmarks/ci/generate_ground_truth.py | 5 +- benchmarks/ci/ground_truth.json | 4 ++ benchmarks/ci/test_bench_ci_moss.py | 73 ++++++++++++++++++++++++-- 7 files changed, 122 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8a3a6bcd..980df371 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,6 +3,14 @@ name: Benchmark permissions: contents: read +# All benchmark runs share one Moss project and the deterministic +# benchmark-ci- index, so concurrent jobs could race on first index +# creation and contaminate each other's latency numbers with cross-job +# load. Serialize globally; don't cancel a run that is already measuring. +concurrency: + group: moss-benchmark + cancel-in-progress: false + on: push: branches: [main] diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index 5a55570f..8beabd3d 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -93,11 +93,24 @@ The benchmark index name is derived from a content signature — `benchmark-ci-` where the hash covers the model id, `DOC_COUNT`, and the exact corpus slice being indexed. If any of those inputs change, the name changes and the index is rebuilt from the current corpus, so a stale -remote index can never be silently benchmarked. `ground_truth.json` embeds -the same signature; the recall test **fails** (with a regenerate hint) when -the ground truth was generated from different inputs. Set `MOSS_INDEX_NAME` +remote index can never be silently benchmarked. Set `MOSS_INDEX_NAME` to override the derived name (this bypasses the staleness protection for -the index itself; the ground-truth signature check still applies). +the index itself; the compatibility checks below still apply). + +Three compatibility checks keep every comparison honest, and each **fails** +(never skips) on mismatch: + +- `ground_truth.json` embeds the corpus/model **signature** — recall is not + evaluated against ground truth generated from different inputs. +- `ground_truth.json` also embeds a **query-set hash** — adding, removing, + or editing `QUERIES` without regenerating trips it. +- `baseline.json`'s config (signature, query-set hash, doc count, rounds, + top_k) must match the current run's config — the regression guard refuses + to compare against a baseline captured under different benchmark inputs. + +Runs are serialized via a GitHub Actions `concurrency` group (without +cancelling in-progress runs): all jobs share one Moss project, so parallel +runs could race on index creation and contaminate each other's latency. ## Credentials policy diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json index 4d645064..ee51c631 100644 --- a/benchmarks/ci/baseline.json +++ b/benchmarks/ci/baseline.json @@ -1,7 +1,7 @@ { "commit": "f4732a1", "timestamp": "2026-07-20T05:35:00+00:00", - "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero — latency is hardware-dependent, so the guard skips until a baseline captured on CI runners replaces this file (run the Benchmark workflow, download the benchmark-results- artifact, commit it here).", + "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero \u2014 latency is hardware-dependent, so the guard skips until a baseline captured on CI runners replaces this file (run the Benchmark workflow, download the benchmark-results- artifact, commit it here).", "latency_ms": { "p50": 0, "p95": 0, @@ -19,6 +19,9 @@ "doc_count": 1000, "query_rounds": 20, "warmup_rounds": 3, - "top_k_latency": 5 + "top_k_latency": 5, + "signature": "73d5e83176f7", + "query_set_hash": "70d932a5a939", + "query_count": 15 } -} +} \ No newline at end of file diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py index e50a0a09..dd6f8625 100644 --- a/benchmarks/ci/bench_queries.py +++ b/benchmarks/ci/bench_queries.py @@ -45,6 +45,20 @@ def index_name_for(signature: str) -> str: """Benchmark index name derived from the corpus/model signature.""" return f"{INDEX_NAME_PREFIX}-{signature}" + +def query_set_hash() -> str: + """Short hash of the benchmark query set. + + Stored in ``ground_truth.json`` and in every results file's config so + the recall test and the regression guard can detect a query set that + drifted from the one used at generation/baseline time. + """ + h = hashlib.sha256() + for q in QUERIES: + h.update(q.encode()) + h.update(b"\x00") + return h.hexdigest()[:12] + QUERIES = [ "neural network training data", "anomaly detection patterns", diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py index a2737a5b..dcccabf2 100644 --- a/benchmarks/ci/generate_ground_truth.py +++ b/benchmarks/ci/generate_ground_truth.py @@ -41,6 +41,7 @@ corpus_signature, index_name_for, load_corpus_slice, + query_set_hash, ) load_dotenv() @@ -122,8 +123,10 @@ async def main(recreate: bool) -> None: "index_name": index_name, "doc_count": DOC_COUNT, # Validated by the benchmark tests: recall is only evaluated when the - # current corpus/model signature matches the one used at generation. + # current corpus/model signature AND query set match the ones used + # at generation. "signature": signature, + "query_set": {"hash": query_set_hash(), "count": len(QUERIES)}, "queries": ground_truth, } diff --git a/benchmarks/ci/ground_truth.json b/benchmarks/ci/ground_truth.json index 641da94f..0216e608 100644 --- a/benchmarks/ci/ground_truth.json +++ b/benchmarks/ci/ground_truth.json @@ -4,6 +4,10 @@ "index_name": "benchmark-ci-73d5e83176f7", "doc_count": 1000, "signature": "73d5e83176f7", + "query_set": { + "hash": "70d932a5a939", + "count": 15 + }, "queries": { "neural network training data": [ "doc_113", diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 0954a02b..eefee002 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -37,6 +37,7 @@ corpus_signature, index_name_for, load_corpus_slice, + query_set_hash, ) load_dotenv() @@ -101,6 +102,50 @@ def _git_sha() -> str: return "unknown" +def _missing_required_input(message: str): + """Handle a missing required benchmark input (corpus, ground truth, …). + + Skip on fork PRs (``ALLOW_BENCHMARK_SKIP=1``) and local runs, but FAIL + in trusted CI — a missing required input must not turn the benchmark + workflow into a green no-op. + """ + if os.getenv("ALLOW_BENCHMARK_SKIP") == "1": + pytest.skip(f"{message} — fork PR, skipping benchmarks") + if os.getenv("CI"): + pytest.fail(f"{message} — required input missing in a trusted CI run") + pytest.skip(message) + + +# Config keys that must match between the current run and the baseline for a +# regression comparison to be meaningful. index_name is excluded: it may be +# overridden via MOSS_INDEX_NAME without changing what is measured. +BASELINE_COMPAT_KEYS = ( + "signature", + "query_set_hash", + "doc_count", + "query_rounds", + "warmup_rounds", + "top_k_latency", +) + + +def _assert_baseline_compatible(baseline: dict, benchmark_results: dict) -> None: + """Fail when the baseline was captured under different benchmark inputs. + + Comparing against a baseline built from another corpus/model, query set, + or measurement config silently masks (or fabricates) regressions. + """ + current = {k: benchmark_results.get("config", {}).get(k) for k in BASELINE_COMPAT_KEYS} + base = {k: baseline.get("config", {}).get(k) for k in BASELINE_COMPAT_KEYS} + if base != current: + diffs = {k: (base[k], current[k]) for k in BASELINE_COMPAT_KEYS if base[k] != current[k]} + pytest.fail( + "baseline.json is not comparable to this run — config mismatch " + f"(baseline vs current): {diffs}. Regenerate the baseline via the " + "Benchmark workflow with update_baseline=true and commit the artifact." + ) + + # --------------------------------------------------------------------------- # Session-scoped fixtures # --------------------------------------------------------------------------- @@ -111,7 +156,7 @@ def corpus_slice() -> list[dict]: """The exact corpus slice the benchmark index is built from.""" corpus_path = CI_DIR.parent / "bench_100k_docs.json" if not corpus_path.exists(): - pytest.skip(f"Corpus file not found: {corpus_path}") + _missing_required_input(f"Corpus file not found: {corpus_path}") return load_corpus_slice(corpus_path) @@ -187,7 +232,7 @@ def ground_truth(corpus_sig) -> dict[str, list[str]]: """ gt_path = CI_DIR / "ground_truth.json" if not gt_path.exists(): - pytest.skip(f"Ground truth file not found: {gt_path}") + _missing_required_input(f"Ground truth file not found: {gt_path}") with open(gt_path) as f: data = json.load(f) gt_sig = data.get("signature") @@ -198,6 +243,14 @@ def ground_truth(corpus_sig) -> dict[str, list[str]]: "model changed since generation. Regenerate with: " "python benchmarks/ci/generate_ground_truth.py --recreate" ) + gt_query_hash = data.get("query_set", {}).get("hash") + if gt_query_hash != query_set_hash(): + pytest.fail( + f"ground_truth.json query-set hash {gt_query_hash!r} does not match " + f"the current query set ({query_set_hash()!r}) — QUERIES changed " + "since generation. Regenerate with: " + "python benchmarks/ci/generate_ground_truth.py" + ) return data.get("queries", {}) @@ -216,6 +269,8 @@ def benchmark_results(corpus_sig) -> dict: "warmup_rounds": WARMUP_ROUNDS, "top_k_latency": TOP_K_LATENCY, "signature": corpus_sig, + "query_set_hash": query_set_hash(), + "query_count": len(QUERIES), "index_name": os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig), }, "latency_ms": {}, @@ -347,8 +402,12 @@ def test_no_latency_regression(self, request, benchmark_results): baseline_path = request.config.getoption("--baseline-file") threshold = request.config.getoption("--latency-threshold") - if not baseline_path or not Path(baseline_path).exists(): + if not baseline_path: pytest.skip("No baseline file provided — skipping regression check") + if not Path(baseline_path).exists(): + # An explicitly requested baseline that is absent is a config + # error, not an optional feature — never a green skip. + pytest.fail(f"--baseline-file was provided but does not exist: {baseline_path}") with open(baseline_path) as f: baseline = json.load(f) @@ -359,6 +418,8 @@ def test_no_latency_regression(self, request, benchmark_results): if baseline_p95 is None or current_p95 is None: pytest.skip("Latency data not yet available — run latency test first") + _assert_baseline_compatible(baseline, benchmark_results) + if baseline_p95 == 0: pytest.skip("Baseline p95 is zero — cannot compute regression ratio") @@ -380,8 +441,10 @@ def test_no_recall_regression(self, request, benchmark_results): baseline_path = request.config.getoption("--baseline-file") threshold = request.config.getoption("--recall-threshold") - if not baseline_path or not Path(baseline_path).exists(): + if not baseline_path: pytest.skip("No baseline file provided — skipping regression check") + if not Path(baseline_path).exists(): + pytest.fail(f"--baseline-file was provided but does not exist: {baseline_path}") with open(baseline_path) as f: baseline = json.load(f) @@ -392,6 +455,8 @@ def test_no_recall_regression(self, request, benchmark_results): if baseline_recall is None or current_recall is None: pytest.skip("Recall data not yet available — run recall test first") + _assert_baseline_compatible(baseline, benchmark_results) + drop = baseline_recall - current_recall print("\n Recall regression check:") From 16b03437dbf9e8c692a5eb16424399d5517fa8cb Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:10:46 +0530 Subject: [PATCH 10/17] ci: retry pip install on transient PyPI download failures Hosted runners occasionally drop the connection mid-wheel-download (urllib3 IncompleteRead / ProtocolError), and pip does not resume or retry a broken stream. Retry the requirements install up to 3 times with a short backoff before failing the job. Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 980df371..babcc038 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -37,7 +37,18 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r benchmarks/ci/requirements.txt + # PyPI downloads occasionally break mid-stream on hosted runners + # (IncompleteRead / ProtocolError), and pip does not resume a + # partial wheel download — retry the whole install a few times. + for attempt in 1 2 3; do + if pip install -r benchmarks/ci/requirements.txt; then + exit 0 + fi + echo "pip install failed (attempt ${attempt}/3) — retrying in 15s" + sleep 15 + done + echo "pip install failed after 3 attempts" + exit 1 - name: Run benchmark suite env: From 1814b155a624ffba2290d8cd00222aa121372067 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:15:29 +0530 Subject: [PATCH 11/17] fix: guard --recreate against deleting non-benchmark indexes; fail on zero latency baseline Addresses two review findings: - BLOCKING: --recreate deleted whatever MOSS_INDEX_NAME pointed at, so a developer with a shared or production Moss project in their env could destroy a non-benchmark index. Deletion is now guarded three ways: the derived benchmark-ci- name deletes without confirmation, an overridden name inside the benchmark-ci-* namespace additionally requires the new --force flag, and names outside that namespace are refused even with --force. - CONSIDER: the checked-in zero latency baseline made test_no_latency_regression skip on every run, leaving the latency guard silently inactive until someone manually armed it. A zero baseline now FAILS comparison runs with the arming procedure in the message. update_baseline dispatch runs are unaffected (they do not pass --baseline-file), and fork PRs skip earlier at the credentials gate, so the failure lands exactly on trusted runs that should be enforcing the guard. baseline.json note and README updated; the first trusted CI run's artifact is the natural baseline to commit. Co-Authored-By: Claude Fable 5 --- benchmarks/ci/README.md | 10 +++++++ benchmarks/ci/baseline.json | 2 +- benchmarks/ci/generate_ground_truth.py | 38 ++++++++++++++++++++++++-- benchmarks/ci/test_bench_ci_moss.py | 11 +++++++- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index 8beabd3d..4b958916 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -50,6 +50,12 @@ expected document IDs to `ground_truth.json`. Commit the updated file. After a corpus or model change, pass `--recreate` so the index is rebuilt from the current corpus before the ground truth is captured. +`--recreate` deletion is guarded: only the derived `benchmark-ci-` +index is deleted without confirmation. If `MOSS_INDEX_NAME` overrides the +name, deletion additionally requires `--force`, and indexes outside the +`benchmark-ci-*` namespace are never deleted — so a stray environment +variable pointing at a shared or production index cannot be destroyed. + > **Note**: the ground truth is a *ranking-stability reference* generated by > Moss itself at a known-good commit — not an independent relevance judgment. > The recall gate detects changes in retrieval behavior; an intentional @@ -63,6 +69,10 @@ The harness compares the current run's metrics against `baseline.json`: - **Latency**: Fails if P95 increases by more than the threshold (default 20%) - **Recall**: Fails if Recall@5 drops by more than the threshold (default 5pp) +A zero latency baseline **fails** comparison runs rather than skipping — the +guard cannot stay silently inactive. To arm it, commit a CI-captured +baseline (see "Updating the baseline" below). + Thresholds are configurable via CLI flags: ```bash diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json index ee51c631..9d888a98 100644 --- a/benchmarks/ci/baseline.json +++ b/benchmarks/ci/baseline.json @@ -1,7 +1,7 @@ { "commit": "f4732a1", "timestamp": "2026-07-20T05:35:00+00:00", - "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero \u2014 latency is hardware-dependent, so the guard skips until a baseline captured on CI runners replaces this file (run the Benchmark workflow, download the benchmark-results- artifact, commit it here).", + "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero and the latency guard FAILS on zero baselines in comparison runs \u2014 the first trusted CI run will be red until a CI-captured baseline is committed: run the Benchmark workflow (update_baseline=true also works), download the benchmark-results- artifact, and commit it here. Latency baselines must come from CI runners; numbers from other hardware are not comparable.", "latency_ms": { "p50": 0, "p95": 0, diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py index dcccabf2..e49d8b4b 100644 --- a/benchmarks/ci/generate_ground_truth.py +++ b/benchmarks/ci/generate_ground_truth.py @@ -36,6 +36,7 @@ from bench_queries import ( DOC_COUNT, + INDEX_NAME_PREFIX, MODEL_ID, QUERIES, corpus_signature, @@ -70,7 +71,32 @@ async def _create_index(client, index_name: str, corpus_slice: list[dict]) -> No print(f"Created index '{index_name}' with {result.doc_count} docs") -async def main(recreate: bool) -> None: +def _guard_deletion(index_name: str, derived_name: str, force: bool) -> None: + """Refuse to delete indexes that are not clearly benchmark-owned. + + MOSS_INDEX_NAME is a documented override, so a developer whose + environment points at a shared or production Moss project could + otherwise aim --recreate at a non-benchmark index and destroy it. + """ + if index_name == derived_name: + return # the derived benchmark index — always safe to recreate + if not index_name.startswith(f"{INDEX_NAME_PREFIX}-"): + print( + f"Error: refusing to delete index '{index_name}' — it is outside " + f"the benchmark namespace ('{INDEX_NAME_PREFIX}-*'). Unset " + "MOSS_INDEX_NAME or point it at a benchmark index." + ) + sys.exit(1) + if not force: + print( + f"Error: MOSS_INDEX_NAME overrides the derived name " + f"('{index_name}' != '{derived_name}'). Pass --force to confirm " + "deleting the overridden benchmark index." + ) + sys.exit(1) + + +async def main(recreate: bool, force: bool) -> None: from moss import MossClient, QueryOptions project_id = os.getenv("MOSS_PROJECT_ID") @@ -94,6 +120,7 @@ async def main(recreate: bool) -> None: existing = {idx.name for idx in await client.list_indexes()} if index_name in existing and recreate: + _guard_deletion(index_name, index_name_for(signature), force) print(f"--recreate: deleting existing index '{index_name}'") await client.delete_index(index_name) existing.discard(index_name) @@ -147,5 +174,12 @@ async def main(recreate: bool) -> None: "querying. Required after a corpus or embedding-model change so the " "ground truth reflects the current data.", ) + parser.add_argument( + "--force", + action="store_true", + help="Confirm --recreate deletion when MOSS_INDEX_NAME overrides the " + "derived index name. Only benchmark-namespace indexes " + "(benchmark-ci-*) can be deleted even with this flag.", + ) args = parser.parse_args() - asyncio.run(main(recreate=args.recreate)) + asyncio.run(main(recreate=args.recreate, force=args.force)) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index eefee002..f8b925cf 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -421,7 +421,16 @@ def test_no_latency_regression(self, request, benchmark_results): _assert_baseline_compatible(baseline, benchmark_results) if baseline_p95 == 0: - pytest.skip("Baseline p95 is zero — cannot compute regression ratio") + # A zero baseline means the latency guard has never been armed. + # Skipping here would let every run pass with the guard silently + # inactive — fail instead, with the arming procedure. + pytest.fail( + "Baseline p95 is zero — the latency guard is not armed. Run the " + "Benchmark workflow with update_baseline=true, download the " + "benchmark-results- artifact, and commit it as " + "benchmarks/ci/baseline.json (values must come from CI runners; " + "this run's artifact works too)." + ) regression = (current_p95 - baseline_p95) / baseline_p95 From 85618ba9a5a30721536d507b6dcd278411d9dd62 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:22:16 +0530 Subject: [PATCH 12/17] ci: retrigger after transient PyPI download failure in python-sdk-test (3.14) From 3e6ac46a02e2af53f2f6be990a0613e0eb58cbbc Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:29:44 +0530 Subject: [PATCH 13/17] fix: explicit unarmed marker for the placeholder latency baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fail-on-zero behavior made every trusted run red until a CI-captured baseline was committed — and this PR cannot ship one, since it runs from a fork without secrets. Per review, the comparison is now gated on the baseline file itself: - baseline.json declares latency_guard: unarmed — the latency test skips with a loud arming message while the recall guard stays active (dropping --baseline-file entirely would have disabled recall too) - committing a CI-captured benchmark-results- artifact as baseline.json arms the guard automatically: artifacts carry no latency_guard flag - a zero p95 WITHOUT the unarmed marker still fails as a misconfigured baseline, and a non-zero p95 WITH the marker fails as inconsistent, so the guard can never be silently inactive by accident Co-Authored-By: Claude Fable 5 --- benchmarks/ci/README.md | 13 +++++++++--- benchmarks/ci/baseline.json | 3 ++- benchmarks/ci/test_bench_ci_moss.py | 32 +++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index 4b958916..14b44ef9 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -69,9 +69,16 @@ The harness compares the current run's metrics against `baseline.json`: - **Latency**: Fails if P95 increases by more than the threshold (default 20%) - **Recall**: Fails if Recall@5 drops by more than the threshold (default 5pp) -A zero latency baseline **fails** comparison runs rather than skipping — the -guard cannot stay silently inactive. To arm it, commit a CI-captured -baseline (see "Updating the baseline" below). +The latency guard arms itself from the baseline file: + +- The checked-in placeholder declares `"latency_guard": "unarmed"` — the + latency test skips with a loud message while recall remains guarded, so + trusted runs are not red before the first baseline exists. +- To arm the guard, download the `benchmark-results-` artifact from a + trusted CI run and commit it as `baseline.json`. Artifacts carry no + `latency_guard` flag, so committing one arms the guard automatically. +- A zero p95 **without** the unarmed flag fails the run as a misconfigured + baseline — the guard can never be silently inactive by accident. Thresholds are configurable via CLI flags: diff --git a/benchmarks/ci/baseline.json b/benchmarks/ci/baseline.json index 9d888a98..7e58e7da 100644 --- a/benchmarks/ci/baseline.json +++ b/benchmarks/ci/baseline.json @@ -1,7 +1,8 @@ { "commit": "f4732a1", "timestamp": "2026-07-20T05:35:00+00:00", - "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are intentionally zero and the latency guard FAILS on zero baselines in comparison runs \u2014 the first trusted CI run will be red until a CI-captured baseline is committed: run the Benchmark workflow (update_baseline=true also works), download the benchmark-results- artifact, and commit it here. Latency baselines must come from CI runners; numbers from other hardware are not comparable.", + "latency_guard": "unarmed", + "_note": "Recall values are measured (hardware-independent) so the recall guard is active. Latency values are zero and 'latency_guard: unarmed' marks this file as the explicit placeholder: the latency regression test skips (loudly) instead of failing every trusted run. To arm the latency guard, run the Benchmark workflow on a trusted ref, download the benchmark-results- artifact, and commit it as this file \u2014 artifacts carry no latency_guard flag, so committing one arms the guard automatically. A zero p95 WITHOUT the unarmed flag fails the run as a misconfigured baseline. Latency baselines must come from CI runners; numbers from other hardware are not comparable.", "latency_ms": { "p50": 0, "p95": 0, diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index f8b925cf..3215a18b 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -420,16 +420,32 @@ def test_no_latency_regression(self, request, benchmark_results): _assert_baseline_compatible(baseline, benchmark_results) + if baseline.get("latency_guard") == "unarmed": + # The checked-in placeholder declares itself unarmed: latency + # baselines must come from CI runners, and none has been captured + # yet. Skip loudly (recall is still guarded) instead of failing + # every trusted run until the first artifact lands. + if baseline_p95 != 0: + pytest.fail( + "baseline.json marks latency_guard as 'unarmed' but contains " + "a non-zero p95 — remove the latency_guard flag to arm the " + "guard." + ) + pytest.skip( + "LATENCY GUARD NOT ARMED — baseline.json is the explicit " + "placeholder (latency_guard: unarmed). To arm it: download the " + "benchmark-results- artifact from a trusted CI run and " + "commit it as benchmarks/ci/baseline.json (the artifact carries " + "no latency_guard flag, so committing it arms the guard)." + ) + if baseline_p95 == 0: - # A zero baseline means the latency guard has never been armed. - # Skipping here would let every run pass with the guard silently - # inactive — fail instead, with the arming procedure. + # Zero without the explicit unarmed marker is a misconfigured + # baseline, not a placeholder — never a silent pass. pytest.fail( - "Baseline p95 is zero — the latency guard is not armed. Run the " - "Benchmark workflow with update_baseline=true, download the " - "benchmark-results- artifact, and commit it as " - "benchmarks/ci/baseline.json (values must come from CI runners; " - "this run's artifact works too)." + "Baseline p95 is zero but baseline.json does not declare " + "latency_guard: unarmed — the baseline is misconfigured. Commit " + "a CI-captured baseline or restore the explicit placeholder." ) regression = (current_p95 - baseline_p95) / baseline_p95 From 062eebce02bddf4a552adf46b22ea2ae7880afad Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:44:01 +0530 Subject: [PATCH 14/17] fix: regression guards fail in trusted CI when measurement data is missing The guards previously skipped on absent latency/recall data, so a run with --baseline-file could go green without ever measuring anything (e.g. a random-ordering plugin collecting a guard before the measurement tests, or measurement crashing in a way that left results empty). Both guards now route missing measurement data through the shared skip-or-fail policy: skip on fork PRs and local runs where measurement legitimately cannot happen, FAIL in trusted CI where a silently passing regression gate is a green no-op. The measure-guard-write ordering comment updated to match: random ordering now fails loudly in trusted CI by design. Verified: local+baseline skips, CI=true without measurements fails both guards, fork PR skips. Co-Authored-By: Claude Fable 5 --- benchmarks/ci/test_bench_ci_moss.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 3215a18b..20d05480 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -103,16 +103,16 @@ def _git_sha() -> str: def _missing_required_input(message: str): - """Handle a missing required benchmark input (corpus, ground truth, …). + """Handle a missing benchmark prerequisite (inputs, measurement data, …). Skip on fork PRs (``ALLOW_BENCHMARK_SKIP=1``) and local runs, but FAIL - in trusted CI — a missing required input must not turn the benchmark + in trusted CI — a missing prerequisite must not turn the benchmark workflow into a green no-op. """ if os.getenv("ALLOW_BENCHMARK_SKIP") == "1": pytest.skip(f"{message} — fork PR, skipping benchmarks") if os.getenv("CI"): - pytest.fail(f"{message} — required input missing in a trusted CI run") + pytest.fail(f"{message} — must not silently pass in a trusted CI run") pytest.skip(message) @@ -280,9 +280,11 @@ def benchmark_results(corpus_sig) -> dict: # --------------------------------------------------------------------------- # Tests — pytest collects these in declaration order (measure → guard → -# write). The ordering is a soft dependency only: the guard and writer -# degrade gracefully (skip / write partial results) if measurement data is -# missing, so a random-ordering plugin breaks nothing, it just skips checks. +# write). Locally and on fork PRs the guards degrade gracefully (skip) when +# measurement data is missing. In trusted CI the guards FAIL on missing +# measurement data — a regression gate that silently passes without +# measurements is a green no-op — so a random-ordering plugin that runs a +# guard before the measurement tests will fail loudly there, by design. # --------------------------------------------------------------------------- @@ -416,7 +418,12 @@ def test_no_latency_regression(self, request, benchmark_results): current_p95 = benchmark_results.get("latency_ms", {}).get("p95") if baseline_p95 is None or current_p95 is None: - pytest.skip("Latency data not yet available — run latency test first") + # A baseline comparison was requested but there is nothing to + # compare: fine when measurement legitimately skipped (fork PR / + # local run without credentials), a red flag in trusted CI. + _missing_required_input( + "Latency measurement data missing — the latency test did not run" + ) _assert_baseline_compatible(baseline, benchmark_results) @@ -478,7 +485,9 @@ def test_no_recall_regression(self, request, benchmark_results): current_recall = benchmark_results.get("recall", {}).get("recall_at_5") if baseline_recall is None or current_recall is None: - pytest.skip("Recall data not yet available — run recall test first") + _missing_required_input( + "Recall measurement data missing — the recall test did not run" + ) _assert_baseline_compatible(baseline, benchmark_results) From bf8f11645a9bf410d2e2e52a0dfb245192099d90 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Tue, 28 Jul 2026 17:53:58 +0530 Subject: [PATCH 15/17] fix: guard recall@10 alongside recall@5 in the regression check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite computes and stores recall_at_10 in both results and the baseline, but the guard only compared recall_at_5 — a change that preserves the top 5 while dropping documents ranked 6-10 would pass CI with recall@10 silently regressed. The guard now loops over both metrics with the same threshold, reports each in the log, and lists every regressed metric in the failure message. conftest help text and README updated to match. Co-Authored-By: Claude Fable 5 --- benchmarks/ci/README.md | 3 +- benchmarks/ci/conftest.py | 4 +-- benchmarks/ci/test_bench_ci_moss.py | 47 +++++++++++++++++------------ 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index 14b44ef9..6b402eef 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -67,7 +67,8 @@ variable pointing at a shared or production index cannot be destroyed. The harness compares the current run's metrics against `baseline.json`: - **Latency**: Fails if P95 increases by more than the threshold (default 20%) -- **Recall**: Fails if Recall@5 drops by more than the threshold (default 5pp) +- **Recall**: Fails if Recall@5 **or** Recall@10 drops by more than the + threshold (default 5pp) The latency guard arms itself from the baseline file: diff --git a/benchmarks/ci/conftest.py b/benchmarks/ci/conftest.py index cd4a7342..59953b56 100644 --- a/benchmarks/ci/conftest.py +++ b/benchmarks/ci/conftest.py @@ -31,6 +31,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--recall-threshold", type=float, default=0.05, - help="Max allowed absolute decrease in recall@5 vs baseline " - "(default: 0.05 = 5 percentage points)", + help="Max allowed absolute decrease in recall@5 and recall@10 vs " + "baseline (default: 0.05 = 5 percentage points)", ) diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 20d05480..420e97b5 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -481,29 +481,38 @@ def test_no_recall_regression(self, request, benchmark_results): with open(baseline_path) as f: baseline = json.load(f) - baseline_recall = baseline.get("recall", {}).get("recall_at_5") - current_recall = benchmark_results.get("recall", {}).get("recall_at_5") - - if baseline_recall is None or current_recall is None: - _missing_required_input( - "Recall measurement data missing — the recall test did not run" - ) + # Guard every recall metric the suite records — checking only + # recall@5 would let a change that preserves the top 5 but drops + # documents ranked 6-10 pass while recall@10 regresses. + recall_pairs: dict[str, tuple[float, float]] = {} + for key in ("recall_at_5", "recall_at_10"): + baseline_val = baseline.get("recall", {}).get(key) + current_val = benchmark_results.get("recall", {}).get(key) + if baseline_val is None or current_val is None: + _missing_required_input( + f"Recall measurement data missing for {key} — " + "the recall test did not run" + ) + recall_pairs[key] = (baseline_val, current_val) _assert_baseline_compatible(baseline, benchmark_results) - drop = baseline_recall - current_recall - + failures: list[str] = [] print("\n Recall regression check:") - print(f" Baseline Recall@5 : {baseline_recall:.4f}") - print(f" Current Recall@5 : {current_recall:.4f}") - print(f" Drop : {drop:+.4f}") - print(f" Threshold : {threshold:.4f}") - - assert drop <= threshold, ( - f"Recall@5 dropped by {drop:.4f} " - f"(baseline={baseline_recall:.4f}, current={current_recall:.4f}, " - f"threshold={threshold:.4f})" - ) + print(f" Threshold: {threshold:.4f}") + for key, (baseline_val, current_val) in recall_pairs.items(): + drop = baseline_val - current_val + print( + f" {key}: baseline={baseline_val:.4f} " + f"current={current_val:.4f} drop={drop:+.4f}" + ) + if drop > threshold: + failures.append( + f"{key} dropped by {drop:.4f} (baseline={baseline_val:.4f}, " + f"current={current_val:.4f}, threshold={threshold:.4f})" + ) + + assert not failures, "Recall regression: " + "; ".join(failures) class TestWriteResults: From ad59ba02a055a6b9035c9bda827df83600ada638 Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Wed, 29 Jul 2026 09:13:50 +0530 Subject: [PATCH 16/17] Address review: exercise index build path per change; write artifact in sessionfinish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Index staleness (blocking finding): the index name now embeds a build fingerprint (installed SDK/bindings versions + Python SDK source tree hash) alongside the corpus/model signature. A PR that changes create_index, document serialization, or the index/model build path gets a new index name, forcing a rebuild that exercises that path — while ground truth stays keyed to data inputs only, so an SDK change that shifts rankings surfaces as a recall regression instead of passing against stale embeddings. generate_ground_truth.py gains --prune to clean up superseded benchmark-namespace indexes. Results artifact: serialization moved from a TestWriteResults test to pytest_sessionfinish in conftest.py, so the artifact is written even under -x/--maxfail, ordering plugins, or a failing regression guard, and a stub documents sessions where nothing was measured. Also satisfies the extended ruff rule set on latest ruff (import sorting, no blocking open() in async code, narrow exception in _git_sha, executable bit on the generator script). Co-Authored-By: Claude Fable 5 --- benchmarks/ci/README.md | 28 +++++++++--- benchmarks/ci/bench_queries.py | 37 +++++++++++++-- benchmarks/ci/conftest.py | 36 ++++++++++++++- benchmarks/ci/generate_ground_truth.py | 53 ++++++++++++++++------ benchmarks/ci/test_bench_ci_moss.py | 63 ++++++++++++++------------ 5 files changed, 164 insertions(+), 53 deletions(-) mode change 100644 => 100755 benchmarks/ci/generate_ground_truth.py diff --git a/benchmarks/ci/README.md b/benchmarks/ci/README.md index 6b402eef..8a355a7b 100644 --- a/benchmarks/ci/README.md +++ b/benchmarks/ci/README.md @@ -107,13 +107,27 @@ change), update the baseline: ## Index naming and staleness protection -The benchmark index name is derived from a content signature — -`benchmark-ci-` where the hash covers the model id, `DOC_COUNT`, and -the exact corpus slice being indexed. If any of those inputs change, the -name changes and the index is rebuilt from the current corpus, so a stale -remote index can never be silently benchmarked. Set `MOSS_INDEX_NAME` -to override the derived name (this bypasses the staleness protection for -the index itself; the compatibility checks below still apply). +The benchmark index name is derived from two content hashes — +`benchmark-ci--`: + +- **data signature** covers the model id, `DOC_COUNT`, and the exact corpus + slice being indexed; +- **build fingerprint** covers the installed SDK/bindings versions and the + Python SDK source tree, i.e. the code that builds the index. + +If any of those inputs change, the name changes and the index is rebuilt +from the current corpus by the current code — so a stale remote index can +never be silently benchmarked, and a PR that changes `create_index`, +document serialization, or the index/model build path always exercises that +path (an embedding change then surfaces as a recall regression instead of +passing against old embeddings). Set `MOSS_INDEX_NAME` to override the +derived name (this bypasses the staleness protection for the index itself; +the compatibility checks below still apply). + +Superseded indexes accumulate in the Moss project as signatures change; +clean them up occasionally with +`python benchmarks/ci/generate_ground_truth.py --prune` (only +`benchmark-ci-*` indexes are ever deleted). Three compatibility checks keep every comparison honest, and each **fails** (never skips) on mismatch: diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py index dd6f8625..bfc232ca 100644 --- a/benchmarks/ci/bench_queries.py +++ b/benchmarks/ci/bench_queries.py @@ -41,9 +41,40 @@ def corpus_signature(docs: list[dict[str, Any]]) -> str: return h.hexdigest()[:12] -def index_name_for(signature: str) -> str: - """Benchmark index name derived from the corpus/model signature.""" - return f"{INDEX_NAME_PREFIX}-{signature}" +def build_fingerprint() -> str: + """Fingerprint of the code path that builds the index. + + Covers the installed SDK/bindings versions and, when running from the + repository, a content hash of the Python SDK source tree. Any change to + the indexing/build path yields a new fingerprint — and therefore a new + index name via ``index_name_for`` — so the benchmark rebuilds the index + and exercises ``create_index``/document serialization instead of loading + an index built by older code (which could pass on stale embeddings). + """ + from importlib.metadata import PackageNotFoundError, version + + h = hashlib.sha256() + for pkg in ("moss", "inferedge-moss", "inferedge-moss-core"): + try: + h.update(f"{pkg}={version(pkg)}".encode()) + except PackageNotFoundError: + pass + sdk_src = Path(__file__).resolve().parents[2] / "sdks" / "python" / "sdk" / "src" + if sdk_src.is_dir(): + for p in sorted(sdk_src.rglob("*.py")): + h.update(str(p.relative_to(sdk_src)).encode()) + h.update(p.read_bytes()) + return h.hexdigest()[:12] + + +def index_name_for(signature: str, fingerprint: str) -> str: + """Benchmark index name derived from data signature + build fingerprint. + + The name is the index's manifest: it changes whenever the corpus slice, + DOC_COUNT, model, or the SDK build path changes, so a stale remote index + can never be silently reused against mismatched inputs or code. + """ + return f"{INDEX_NAME_PREFIX}-{signature}-{fingerprint}" def query_set_hash() -> str: diff --git a/benchmarks/ci/conftest.py b/benchmarks/ci/conftest.py index 59953b56..3c3fa05a 100644 --- a/benchmarks/ci/conftest.py +++ b/benchmarks/ci/conftest.py @@ -1,9 +1,14 @@ """Pytest configuration for the CI benchmark suite. Adds custom CLI flags so the harness can be invoked as a standard pytest -run with configurable output paths and regression thresholds. +run with configurable output paths and regression thresholds, and writes +the results artifact at session end. """ +import json +from datetime import datetime, timezone +from pathlib import Path + import pytest @@ -34,3 +39,32 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Max allowed absolute decrease in recall@5 and recall@10 vs " "baseline (default: 0.05 = 5 percentage points)", ) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Serialize benchmark results after the run. + + Writing here (rather than in a test that must be collected last) means + the artifact is emitted regardless of test ordering, ``-x``/``--maxfail`` + early exits, or a failing regression guard — CI's ``if: always()`` + artifact upload always has real data to capture. + """ + config = session.config + if config.option.collectonly: + return + results = getattr(config, "_benchmark_results", None) + if results is None: + # No measurement fixture was ever instantiated (e.g. credentials + # missing and everything skipped). Still emit a stub so the artifact + # documents that nothing was measured instead of silently vanishing. + results = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "note": "no benchmark measurements ran in this session", + "latency_ms": {}, + "recall": {}, + } + output_path = Path(config.getoption("--benchmark-output")) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\nBenchmark results written to: {output_path}") diff --git a/benchmarks/ci/generate_ground_truth.py b/benchmarks/ci/generate_ground_truth.py old mode 100644 new mode 100755 index e49d8b4b..a9bfd902 --- a/benchmarks/ci/generate_ground_truth.py +++ b/benchmarks/ci/generate_ground_truth.py @@ -32,18 +32,18 @@ import sys from pathlib import Path -from dotenv import load_dotenv - from bench_queries import ( DOC_COUNT, INDEX_NAME_PREFIX, MODEL_ID, QUERIES, + build_fingerprint, corpus_signature, index_name_for, load_corpus_slice, query_set_hash, ) +from dotenv import load_dotenv load_dotenv() @@ -96,17 +96,19 @@ def _guard_deletion(index_name: str, derived_name: str, force: bool) -> None: sys.exit(1) -async def main(recreate: bool, force: bool) -> None: +async def main(recreate: bool, force: bool, prune: bool) -> dict: from moss import MossClient, QueryOptions project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") # Same derivation as the benchmark tests: the index name embeds the - # corpus/model signature, so generation and evaluation can never target - # indexes built from different inputs. + # corpus/model signature and the SDK build fingerprint, so generation + # and evaluation can never target indexes built from different inputs + # or by different indexing code. corpus_slice = _corpus_slice() signature = corpus_signature(corpus_slice) - index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(signature) + derived_name = index_name_for(signature, build_fingerprint()) + index_name = os.getenv("MOSS_INDEX_NAME") or derived_name if not project_id or not project_key: print("Error: MOSS_PROJECT_ID and MOSS_PROJECT_KEY must be set.") @@ -120,7 +122,7 @@ async def main(recreate: bool, force: bool) -> None: existing = {idx.name for idx in await client.list_indexes()} if index_name in existing and recreate: - _guard_deletion(index_name, index_name_for(signature), force) + _guard_deletion(index_name, derived_name, force) print(f"--recreate: deleting existing index '{index_name}'") await client.delete_index(index_name) existing.discard(index_name) @@ -157,12 +159,23 @@ async def main(recreate: bool, force: bool) -> None: "queries": ground_truth, } - output_path = Path(__file__).resolve().parent / "ground_truth.json" - with open(output_path, "w") as f: - json.dump(output, f, indent=2) + if prune: + # Old corpus/SDK revisions leave benchmark-ci-* indexes behind; + # remove everything in the benchmark namespace except the index we + # just used. Never touches indexes outside the namespace. + stale = sorted( + n + for n in existing + if n != index_name + and (n == INDEX_NAME_PREFIX or n.startswith(f"{INDEX_NAME_PREFIX}-")) + ) + for n in stale: + print(f"--prune: deleting stale benchmark index '{n}'") + await client.delete_index(n) + if not stale: + print("--prune: no stale benchmark indexes found") - print(f"\nGround truth written to: {output_path}") - print(f"Queries: {len(ground_truth)}") + return output if __name__ == "__main__": @@ -181,5 +194,19 @@ async def main(recreate: bool, force: bool) -> None: "derived index name. Only benchmark-namespace indexes " "(benchmark-ci-*) can be deleted even with this flag.", ) + parser.add_argument( + "--prune", + action="store_true", + help="After generating, delete leftover benchmark-namespace indexes " + "(benchmark-ci-*) from earlier corpus/SDK revisions. Indexes outside " + "the benchmark namespace are never touched.", + ) args = parser.parse_args() - asyncio.run(main(recreate=args.recreate, force=args.force)) + output = asyncio.run(main(recreate=args.recreate, force=args.force, prune=args.prune)) + + output_path = Path(__file__).resolve().parent / "ground_truth.json" + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + + print(f"\nGround truth written to: {output_path}") + print(f"Queries: {len(output['queries'])}") diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 420e97b5..2bab5e38 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -28,17 +28,17 @@ from pathlib import Path import pytest -from dotenv import load_dotenv - from bench_queries import ( DOC_COUNT, MODEL_ID, QUERIES, + build_fingerprint, corpus_signature, index_name_for, load_corpus_slice, query_set_hash, ) +from dotenv import load_dotenv load_dotenv() @@ -66,7 +66,7 @@ def _percentile(values: list[float], p: float) -> float: """Compute the *p*-th percentile from a **sorted** list of values.""" if not values: return 0.0 - idx = max(int(math.ceil(p * len(values))) - 1, 0) + idx = max(math.ceil(p * len(values)) - 1, 0) return values[idx] @@ -98,7 +98,7 @@ def _git_sha() -> str: check=True, ) return result.stdout.strip() - except Exception: + except (subprocess.SubprocessError, OSError): return "unknown" @@ -167,13 +167,21 @@ def corpus_sig(corpus_slice) -> str: @pytest.fixture(scope="session") -def moss_client(corpus_slice, corpus_sig): +def build_fp() -> str: + """Fingerprint of the index build path (SDK versions + source tree).""" + return build_fingerprint() + + +@pytest.fixture(scope="session") +def moss_client(corpus_slice, corpus_sig, build_fp): """Create a MossClient and load the benchmark index once per session. - The index name embeds ``corpus_sig``, so an index built from a different - corpus, DOC_COUNT, or model can never be silently reused — mismatched - inputs produce a different name and the index is (re)created from the - current corpus. + The index name embeds ``corpus_sig`` (corpus/DOC_COUNT/model) AND + ``build_fp`` (SDK versions + source), so an index built from different + data or by different indexing code can never be silently reused — + mismatched inputs produce a different name and the index is (re)created + from the current corpus by the current code, exercising the full + create_index/document-serialization path whenever it changes. """ project_id = os.getenv("MOSS_PROJECT_ID") project_key = os.getenv("MOSS_PROJECT_KEY") @@ -194,10 +202,10 @@ def moss_client(corpus_slice, corpus_sig): # Import lazily (and only once credentials are known to exist) — Moss # native bindings may not be installed in every env. - from moss import MossClient, DocumentInfo + from moss import DocumentInfo, MossClient client = MossClient(project_id, project_key) - index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig) + index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig, build_fp) async def _setup(): # Determine existence explicitly (rather than treating any get_index @@ -255,12 +263,14 @@ def ground_truth(corpus_sig) -> dict[str, list[str]]: @pytest.fixture(scope="session") -def benchmark_results(corpus_sig) -> dict: +def benchmark_results(request, corpus_sig, build_fp) -> dict: """Mutable dict that accumulates results across tests in this session. - The ``test_write_results`` finalizer serializes this to JSON. + Registered on the pytest config so ``pytest_sessionfinish`` (in + conftest.py) serializes it to JSON after the run — regardless of test + ordering, ``-x``/``--maxfail``, or guard failures. """ - return { + results = { "commit": _git_sha(), "timestamp": datetime.now(timezone.utc).isoformat(), "config": { @@ -271,17 +281,23 @@ def benchmark_results(corpus_sig) -> dict: "signature": corpus_sig, "query_set_hash": query_set_hash(), "query_count": len(QUERIES), - "index_name": os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig), + "build_fingerprint": build_fp, + "index_name": os.getenv("MOSS_INDEX_NAME") + or index_name_for(corpus_sig, build_fp), }, "latency_ms": {}, "recall": {}, } + request.config._benchmark_results = results + return results # --------------------------------------------------------------------------- -# Tests — pytest collects these in declaration order (measure → guard → -# write). Locally and on fork PRs the guards degrade gracefully (skip) when -# measurement data is missing. In trusted CI the guards FAIL on missing +# Tests — pytest collects these in declaration order (measure → guard). +# Results serialization happens in conftest.pytest_sessionfinish, so the +# artifact is written even under -x/--maxfail, reordering plugins, or guard +# failures. Locally and on fork PRs the guards degrade gracefully (skip) +# when measurement data is missing. In trusted CI the guards FAIL on missing # measurement data — a regression gate that silently passes without # measurements is a green no-op — so a random-ordering plugin that runs a # guard before the measurement tests will fail loudly there, by design. @@ -513,14 +529,3 @@ def test_no_recall_regression(self, request, benchmark_results): ) assert not failures, "Recall regression: " + "; ".join(failures) - - -class TestWriteResults: - """Serialize benchmark results to JSON (always runs last).""" - - def test_write_results(self, request, benchmark_results): - output_path = Path(request.config.getoption("--benchmark-output")) - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w") as f: - json.dump(benchmark_results, f, indent=2) - print(f"\n Results written to: {output_path}") From 942ac8ad1c781435780f72dcea724f85ca29006e Mon Sep 17 00:00:00 2001 From: Sravan1011 Date: Wed, 29 Jul 2026 09:31:59 +0530 Subject: [PATCH 17/17] Address review: hash installed files not versions; fix loop/import ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build fingerprint (blocking finding): now hashes the on-disk content of every file in the installed moss and inferedge-moss-core distributions (Python sources, data, and native .so/.pyd bindings) instead of just their version strings. A native binding rebuilt without a version bump — e.g. a locally compiled wheel, or a binding change shipped under an unchanged pin — previously kept the old fingerprint and could reuse a stale index; it now changes the file bytes and forces a rebuild. MossClient construction moved inside the moss_client fixture's _setup() coroutine, after _run() has created and installed the shared event loop, so any async initialization the client performs binds to the loop it will actually run on. Made python-dotenv optional in test_bench_ci_moss.py (try/except ModuleNotFoundError) so a repo-wide pytest run that collects this module doesn't hard-fail on an unrelated missing dependency before the credential-based skips even run. Extended ALLOW_BENCHMARK_SKIP to also cover same-repo Dependabot PRs, which GitHub denies normal secrets to just like fork PRs, but which the previous fork-only check would have made fail as if secrets were misconfigured. Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 12 +++++---- benchmarks/ci/bench_queries.py | 42 +++++++++++++++++------------ benchmarks/ci/test_bench_ci_moss.py | 15 +++++++++-- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index babcc038..2abbdc4d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -54,11 +54,13 @@ jobs: env: MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }} MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }} - # Fork PRs cannot read repository secrets, so missing credentials - # are expected there and the suite may skip. On trusted runs - # (push to main, same-repo PRs, manual dispatch) missing secrets - # make the suite FAIL instead of passing as a green no-op. - ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && '1' || '0' }} + # Fork PRs, and same-repo Dependabot PRs (which GitHub also denies + # normal secrets), cannot read repository secrets, so missing + # credentials are expected there and the suite may skip. On other + # trusted runs (push to main, same-repo human PRs, manual + # dispatch) missing secrets make the suite FAIL instead of + # passing as a green no-op. + ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]')) && '1' || '0' }} run: | # Baseline-update runs skip the regression comparison: comparing # against the baseline being replaced would fail the run (and skip diff --git a/benchmarks/ci/bench_queries.py b/benchmarks/ci/bench_queries.py index bfc232ca..4909539a 100644 --- a/benchmarks/ci/bench_queries.py +++ b/benchmarks/ci/bench_queries.py @@ -42,28 +42,36 @@ def corpus_signature(docs: list[dict[str, Any]]) -> str: def build_fingerprint() -> str: - """Fingerprint of the code path that builds the index. - - Covers the installed SDK/bindings versions and, when running from the - repository, a content hash of the Python SDK source tree. Any change to - the indexing/build path yields a new fingerprint — and therefore a new - index name via ``index_name_for`` — so the benchmark rebuilds the index - and exercises ``create_index``/document serialization instead of loading - an index built by older code (which could pass on stale embeddings). + """Fingerprint of every file belonging to the packages that build the index. + + Hashes the on-disk content of every file in the installed ``moss`` and + ``inferedge-moss-core`` distributions — Python sources, data files, and + native bindings (``.so``/``.pyd``) alike — rather than just their + version strings. A rebuilt native binding (e.g. a locally compiled wheel + during development, or a binding change that ships under an unchanged + version pin) still changes the file bytes even when the version string + doesn't, so it isn't missed. Any change to the indexing/build path + yields a new fingerprint — and therefore a new index name via + ``index_name_for`` — so the benchmark rebuilds the index and exercises + ``create_index``/document serialization instead of loading an index + built by older code (which could pass on stale embeddings). """ - from importlib.metadata import PackageNotFoundError, version + from importlib.metadata import PackageNotFoundError, distribution h = hashlib.sha256() - for pkg in ("moss", "inferedge-moss", "inferedge-moss-core"): + for pkg in ("moss", "inferedge-moss-core"): try: - h.update(f"{pkg}={version(pkg)}".encode()) + dist = distribution(pkg) except PackageNotFoundError: - pass - sdk_src = Path(__file__).resolve().parents[2] / "sdks" / "python" / "sdk" / "src" - if sdk_src.is_dir(): - for p in sorted(sdk_src.rglob("*.py")): - h.update(str(p.relative_to(sdk_src)).encode()) - h.update(p.read_bytes()) + continue + h.update(f"{pkg}={dist.version}".encode()) + for rel_path in sorted(dist.files or (), key=str): + try: + data = rel_path.read_binary() + except (FileNotFoundError, IsADirectoryError, OSError): + continue + h.update(str(rel_path).encode()) + h.update(data) return h.hexdigest()[:12] diff --git a/benchmarks/ci/test_bench_ci_moss.py b/benchmarks/ci/test_bench_ci_moss.py index 2bab5e38..dde64557 100644 --- a/benchmarks/ci/test_bench_ci_moss.py +++ b/benchmarks/ci/test_bench_ci_moss.py @@ -38,7 +38,13 @@ load_corpus_slice, query_set_hash, ) -from dotenv import load_dotenv +try: + from dotenv import load_dotenv +except ModuleNotFoundError: # pragma: no cover - optional outside benchmarks/ci + + def load_dotenv() -> None: + return None + load_dotenv() @@ -204,10 +210,15 @@ def moss_client(corpus_slice, corpus_sig, build_fp): # native bindings may not be installed in every env. from moss import DocumentInfo, MossClient - client = MossClient(project_id, project_key) index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig, build_fp) async def _setup(): + # Construct the client inside the coroutine, after _run() has + # created and installed the shared event loop — MossClient may bind + # an async session or call get_event_loop() internally, and must do + # so against the loop it will actually run on. + client = MossClient(project_id, project_key) + # Determine existence explicitly (rather than treating any get_index # failure as "missing") so auth/network errors surface instead of # silently triggering index creation.