From ec57dac504381df44a172fe51dec8a1fca8fa0e4 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 09:25:23 -0600 Subject: [PATCH 01/11] Add design spec for C++ and Python coverage reporting Two-phase plan: Phase 1 lands repo-local coverage (CMake option, dev/coverage.sh, Coverage.yaml, codecov.yml) using gcov + gcovr for C++ and coverage.py for Python, unified in Codecov. Phase 2 lifts the generic pieces into xmsconan once Phase 1 stabilizes. Co-Authored-By: Claude Opus 4.7 --- .../2026-05-08-coverage-reporting-design.md | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-08-coverage-reporting-design.md diff --git a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md new file mode 100644 index 00000000..4bf57f5b --- /dev/null +++ b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md @@ -0,0 +1,260 @@ +# Coverage Reporting Design + +**Date:** 2026-05-08 +**Status:** Approved (design phase) +**Owner:** glarsen@aquaveo.com + +## Goal + +Add unified line-coverage reporting for both the C++ library and Python +bindings of `xmsgrid`. Reports are generated locally for developer iteration +and in CI for accountability, with results published to Codecov. + +The C++ extension is built **once** with coverage instrumentation; both the +ctest and pytest suites run against that single build, and their gcov data is +merged into one C++ report. Python-side coverage is collected separately via +coverage.py. + +## Non-Goals + +- Windows / MSVC coverage. gcov is GCC/Clang only; instrumenting MSVC + requires OpenCppCoverage or clang-cl source-based coverage and is a + separate effort. +- Branch coverage. Line coverage only for now. +- Hard coverage gates that block PRs. Reporting is informational; tightening + is a follow-up. +- Refactoring xmsconan. Phase 1 is repo-local. Generic pieces lift in Phase 2. + +## Architecture + +``` +┌─ Coverage Build (Linux, GCC, Debug) ─────────────────────────┐ +│ cmake -DXMSGRID_ENABLE_COVERAGE=ON → --coverage flags │ +│ builds: libxmsgrid (instrumented) │ +│ C++ test runner (instrumented) │ +│ Python extension _xmsgrid (instrumented) │ +└──────────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ + ┌──────────────────┐ ┌─────────────────────┐ + │ ctest │ │ pytest │ + │ → *.gcda files │ │ → *.gcda (C++) │ + │ (C++ tests) │ │ → .coverage (py) │ + └──────────────────┘ └─────────────────────┘ + │ │ + └─────────────┬──────────────────┘ + ▼ + ┌─────────────────────────────┐ + │ gcovr → cobertura-cpp.xml │ + │ coverage → cobertura-py.xml │ + └─────────────────────────────┘ + ▼ + ┌─────────────────────────────┐ + │ Codecov upload │ + │ flag: cpp │ + │ flag: python │ + └─────────────────────────────┘ +``` + +The crucial property: pybind11-built objects compiled with `--coverage` emit +`.gcda` files when their code paths run, regardless of whether the caller is +C++ or Python. So pytest exercising the bindings credits the underlying C++ +implementation in the C++ report. + +## Components + +### 1. CMake option (in `build.toml` `extra_cmake_text`) + +```cmake +option(XMSGRID_ENABLE_COVERAGE "Build with gcov instrumentation" OFF) +if(XMSGRID_ENABLE_COVERAGE) + if(MSVC) + message(FATAL_ERROR "Coverage build is GCC/Clang only") + endif() + add_compile_options(--coverage -O0 -g) + add_link_options(--coverage) +endif() +``` + +The flag is added to the existing `extra_cmake_text` block — no xmsconan +generator change. Off by default, so non-coverage builds are untouched. + +### 2. `dev/coverage.sh` + +POSIX shell script (Linux + macOS). Single entry point. Steps: + +1. Run the Conan build with the coverage flag: + ``` + python build.py \ + --filter='{"build_type":"Debug"}' \ + --cmake-args='-DXMSGRID_ENABLE_COVERAGE=ON' \ + --wheel-dir wheelhouse + ``` +2. Run the C++ test suite via `ctest` against the instrumented build (the + conan recipe already wires this). +3. Install the freshly-built wheel into a throwaway venv: + ``` + python -m venv .coverage-venv + .coverage-venv/bin/pip install wheelhouse/*.whl pytest pytest-cov + ``` +4. Run the Python tests with coverage: + ``` + .coverage-venv/bin/pytest \ + --cov=xms.grid \ + --cov-report=xml:cov-py.xml \ + --cov-report=html:build/coverage-html-py \ + _package/tests + ``` +5. Aggregate C++ gcov data: + ``` + gcovr --root . \ + --filter 'xmsgrid/' \ + --exclude '.*\.t\.h$' \ + --exclude 'xmsgrid/python/.*' \ + --exclude '_package/tests/.*' \ + --xml cov-cpp.xml \ + --html-details build/coverage-html-cpp/index.html + ``` +6. Print a summary line, e.g. `Coverage: C++ 78.3%, Python 64.1%`. + +The wheel installed in step 3 must be the exact one produced in step 1 — +gcov data files reference build-tree absolute paths, so any rebuild between +the two will desync the `.gcda` / `.gcno` pair. + +A `dev/coverage.bat` is **not** added in Phase 1; Windows coverage is out +of scope. + +### 3. `.github/workflows/Coverage.yaml` + +Hand-maintained, separate from the xmsconan-generated `XmsGrid-CI.yaml`. +Single job, Linux, GCC, Debug: + +- Triggers: `push`, `pull_request`. +- Checks out source, installs xmsconan + Python deps, sets up Conan login. +- Runs `dev/coverage.sh`. +- Uploads `cov-cpp.xml` to Codecov with flag `cpp`. +- Uploads `cov-py.xml` to Codecov with flag `python`. +- Uploads `build/coverage-html-cpp` and `build/coverage-html-py` as + workflow artifacts so a developer can download and browse them. + +The job uses the same Conan secrets as the existing CI workflow and the +new `CODECOV_TOKEN` repo secret. + +### 4. `codecov.yml` + +```yaml +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true +flags: + cpp: + paths: + - xmsgrid/ + carryforward: true + python: + paths: + - _package/xms/ + carryforward: true +ignore: + - "_package/tests/" + - "xmsgrid/**/*.t.h" + - "xmsgrid/python/" +``` + +Both project and patch status are informational — Codecov will comment on +PRs with the numbers but won't fail the check. Tightening is a future +decision once we have a baseline. + +### 5. README update + +Short `## Coverage` section: how to run `dev/coverage.sh`, where the HTML +lands, link to the Codecov project page. + +## Repo Setup (one-time, manual) + +- Add the `xmsgrid` repo to Codecov via the GitHub app. +- Add `CODECOV_TOKEN` as a repository secret in GitHub settings. + +These are operator actions, not code changes; they should be documented in +the implementation plan as prerequisites for the Coverage workflow's first +green run. + +## Phase 2 — Lift to xmsconan + +After Phase 1 ships and stabilizes (a few real PRs through the new +workflow, no broken Codecov runs for a week), lift the generic pieces +into xmsconan in a follow-up PR. + +**Lifted into xmsconan:** + +- The CMake coverage option block — emitted by `xmsconan_gen` into the + generated CMakeLists, gated by a flag in `build.toml`. +- `Coverage.yaml` workflow — generated by `xmsconan_gen` alongside the + existing `-CI.yaml`, gated by the same flag. +- Orchestration shipped as a console entry point: `xmsconan_coverage`, + alongside `xmsconan_gen`, `xmsconan_wheel_repair`, `xmsconan_wheel_deploy`. + The repo-local `dev/coverage.sh` shrinks to a one-liner or is deleted in + favor of documenting `xmsconan_coverage`. +- Default gcovr exclusion patterns: `*.t.h`, `_package/tests/`, + `/python/`. + +**Stays repo-local:** + +- A toggle in `build.toml`: + ```toml + [ci] + enable_coverage = true + ``` +- `codecov.yml` (per-repo thresholds and ignore lists). +- `CODECOV_TOKEN` repo secret. + +**Migration of xmsgrid in Phase 2:** + +1. Bump xmsconan dependency in `build.toml` to the version with coverage + support. +2. Set `enable_coverage = true` in `build.toml`. +3. Replace `dev/coverage.sh` body with a call to `xmsconan_coverage`, or + delete the script and document `xmsconan_coverage` directly. +4. Delete `.github/workflows/Coverage.yaml` — now regenerated. +5. Re-run `xmsconan_gen`. +6. Verify the next CI run produces the same Codecov report. + +**Why two phases:** + +- Phase 1 lets us iterate on script details, gcovr filters, and Codecov + config without an xmsconan release on every change. +- Phase 2 is mechanical: copy proven files into xmsconan templates, add + the `build.toml` knob. + +A feature issue tracking Phase 2 will be filed in the xmsconan repo when +Phase 1 lands. + +## Testing + +- Unit testing the coverage script itself is overkill — it's an + orchestrator. Validation is end-to-end: a successful CI run on a PR + shows non-zero numbers for both flags in the Codecov comment. +- Local validation: run `dev/coverage.sh` on a clean checkout, confirm + HTML reports open and show non-zero coverage for at least one file in + each layer. + +## Risks & Open Questions + +- **Pybind11 coverage instrumentation overhead.** `--coverage` adds + significant build time and binary size. Linux Debug coverage build is + expected to take notably longer than the existing Linux Debug CI job; + this is acceptable since coverage is its own job and runs in parallel. +- **gcda path mismatch.** If pytest is run from a different working + directory than the one where the extension was built, gcov may not + find source files. The script pins `cwd` to the repo root throughout. +- **Codecov free-tier rate limits.** Public repos are unlimited on + uploads; no concern for xmsgrid specifically. +- **Linux-only signal.** Coverage numbers reflect what runs on Linux. For + a math/geometry library this is fine — platform-specific code is + minimal — but anything truly Windows- or Mac-only will show as + uncovered. Acceptable. From b6bda79ddc3f832862d84e42bf2ed3a041d71557 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 09:52:57 -0600 Subject: [PATCH 02/11] Add Phase 1 coverage reporting for C++ and Python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C++ coverage comes from a Debug+testing Conan build instrumented via a new env-var-gated CMake block (XMSGRID_COVERAGE) in build.toml's extra_cmake_text; gcovr scans the resulting build folder in the Conan cache. Python coverage runs separately: the Release+pybind build produces a wheel which dev/coverage.sh installs into a clean venv and exercises with pytest-cov. Both reports upload to Codecov as separate flags via a hand-maintained Coverage workflow that lives alongside the xmsconan-generated CI workflow. The codecov.yml is informational-only — Codecov will not block PRs. Cross-credit (Python tests measuring C++ coverage) is deferred to Phase 2, which will lift the orchestration into xmsconan and address the conanfile constraints (BUILD_TESTING/IS_PYTHON_BUILD mutual exclusion, pybind forced to Release, no pytest-cov in the build venv) that make unification costly to do repo-locally. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/Coverage.yaml | 84 ++++++++++ .gitignore | 5 + README.md | 17 ++ build.toml | 12 ++ codecov.yml | 28 ++++ dev/coverage.sh | 149 ++++++++++++++++++ .../2026-05-08-coverage-reporting-design.md | 144 +++++++++++------ 7 files changed, 387 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/Coverage.yaml create mode 100644 codecov.yml create mode 100755 dev/coverage.sh diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml new file mode 100644 index 00000000..c1a26735 --- /dev/null +++ b/.github/workflows/Coverage.yaml @@ -0,0 +1,84 @@ +# Coverage workflow — hand-maintained, separate from xmsconan-generated CI. +# Lifts to xmsconan in Phase 2. +# +# Required repository secrets: +# CONAN2_USER_SECRET - Conan remote login username +# CONAN2_PASSWORD_SECRET - Conan remote login password +# AQUAPI_USERNAME_SECRET - devpi username for xmsconan install +# AQUAPI_PASSWORD_SECRET - devpi password for xmsconan install +# AQUAPI_URL_DEV - devpi index URL for xmsconan install +# CODECOV_TOKEN - Codecov upload token + +name: Coverage + +on: + push: + pull_request: + +jobs: + coverage: + name: Coverage (Linux, GCC, Debug) + runs-on: ubuntu-latest + + container: + image: ghcr.io/aquaveo/conan-gcc13-py3.13:latest + + env: + LIBRARY_NAME: xmsgrid + XMS_VERSION: '0.0.0' + CONAN_REFERENCE: xmsgrid/0.0.0 + CONAN_ARCHS: x86_64 + CONAN_USERNAME: aquaveo + CONAN_CHANNEL: testing + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN2_USER_SECRET }} + CONAN_PASSWORD: ${{ secrets.CONAN2_PASSWORD_SECRET }} + CONAN_REMOTE_URL: https://conan2.aquaveo.com/artifactory/api/conan/aquaveo-stable + AQUAPI_USERNAME: ${{ secrets.AQUAPI_USERNAME_SECRET }} + AQUAPI_PASSWORD: ${{ secrets.AQUAPI_PASSWORD_SECRET }} + AQUAPI_URL: ${{ secrets.AQUAPI_URL_DEV }} + PYTHON_TARGET_VERSION: '3.13' + RELEASE_PYTHON: 'False' + CTEST_PARALLEL_LEVEL: '8' + + steps: + - name: Checkout Source + uses: actions/checkout@v4 + + - name: Install Python Dependencies + run: | + pip install conan devpi-client wheel gcovr + pip install xmsconan>=2.12.2 -i https://public.aquapi.aquaveo.com/aquaveo/dev/+simple + + - name: Setup Conan + run: xmsconan_conan_setup --remote-url ${{ env.CONAN_REMOTE_URL }} --login + shell: bash + + - name: Run coverage + run: dev/coverage.sh + shell: bash + + - name: Upload C++ coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: cov-cpp.xml + flags: cpp + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + + - name: Upload Python coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: cov-py.xml + flags: python + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + + - name: Upload HTML reports as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-html + path: | + build/coverage-html-cpp + build/coverage-html-py + retention-days: 14 diff --git a/.gitignore b/.gitignore index 1bde17d2..dc2629e7 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,11 @@ coverage_report/ *.profdata *.gcda *.gcno +.coverage-mark +.coverage-venv/ +.coverage +cov-cpp.xml +cov-py.xml # ── Local configuration ─────────────────────── .claude/ diff --git a/README.md b/README.md index 4d740554..cf2f1801 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,20 @@ Documentation [C++ Documentation](https://aquaveo.github.io/xmsgrid/) [Python Documentation](https://aquaveo.github.io/xmsgrid/pydocs) + +Coverage +-------- + +Coverage runs on Linux/macOS via `dev/coverage.sh`. The script runs an +instrumented Debug build with the C++ test suite, then a release wheel build +with pytest in a clean venv. Outputs: + +- `cov-cpp.xml` / `cov-py.xml` — Cobertura XML for upload to Codecov. +- `build/coverage-html-cpp/` and `build/coverage-html-py/` — browsable HTML. + +Prerequisites: `gcovr` (`pip install gcovr`) and a working Conan toolchain +(see Building above). Phase 1 is GCC/Clang-only; Windows MSVC coverage is +not supported. + +CI publishes both flags to Codecov on every push and pull request. Codecov +is informational only — it will not block PRs. diff --git a/build.toml b/build.toml index c491f737..dd01d696 100644 --- a/build.toml +++ b/build.toml @@ -10,6 +10,18 @@ extra_cmake_text = """ if (NOT MSVC) add_compile_options(-ffp-contract=off) endif() + +# Coverage instrumentation, gated on the XMSGRID_COVERAGE environment variable. +# Set XMSGRID_COVERAGE=1 before invoking the build to add gcov flags to all +# targets. dev/coverage.sh is the canonical caller. +if (DEFINED ENV{XMSGRID_COVERAGE} AND NOT \"$ENV{XMSGRID_COVERAGE}\" STREQUAL \"\") + if (MSVC) + message(FATAL_ERROR \"XMSGRID_COVERAGE is GCC/Clang only\") + endif () + message(STATUS \"Coverage instrumentation enabled (XMSGRID_COVERAGE)\") + add_compile_options(--coverage -O0 -g) + add_link_options(--coverage) +endif () """ xms_dependencies = [{ name = "xmscore", version = "7.0.8" }] diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..fb0fbadf --- /dev/null +++ b/codecov.yml @@ -0,0 +1,28 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + +flags: + cpp: + paths: + - xmsgrid/ + carryforward: true + python: + paths: + - _package/xms/ + carryforward: true + +ignore: + - "_package/tests/" + - "xmsgrid/**/*.t.h" + - "xmsgrid/python/" + +comment: + layout: "header, diff, flags, files" + behavior: default + require_changes: false diff --git a/dev/coverage.sh b/dev/coverage.sh new file mode 100755 index 00000000..1b2aa700 --- /dev/null +++ b/dev/coverage.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Generate C++ and Python coverage reports for xmsgrid. +# +# C++ coverage comes from a Debug+testing Conan build instrumented with +# --coverage. ctest runs as part of the build; gcovr scans the resulting +# build folder in the conan cache. +# +# Python coverage comes from a separate pytest run against the built wheel +# in a clean venv with pytest-cov. Cross-credit (Python tests measuring C++ +# code) is not in scope for Phase 1. +# +# Outputs: +# cov-cpp.xml Cobertura, C++ +# cov-py.xml Cobertura, Python +# build/coverage-html-cpp/ Browsable C++ report +# build/coverage-html-py/ Browsable Python report +# +# Requirements: gcc/clang, gcovr, python, conan, xmsconan, gcc tools (gcov). + +set -euo pipefail + +REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +if [[ "$(uname -s)" == MINGW* || "$(uname -s)" == CYGWIN* ]]; then + echo "error: coverage is GCC/Clang-only; Windows is unsupported in Phase 1" >&2 + exit 1 +fi + +# Tool checks. +for tool in python gcovr gcov conan xmsconan_gen; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "error: required tool '$tool' not on PATH" >&2 + exit 1 + fi +done + +XMS_VERSION="${XMS_VERSION:-0.0.0}" + +echo "==> Generating xmsconan build artifacts" +xmsconan_gen --version "$XMS_VERSION" build.toml + +MARK="$REPO_ROOT/.coverage-mark" +trap 'rm -f "$MARK"' EXIT +touch "$MARK" +sleep 1 # Ensure subsequent file mtimes strictly exceed the marker. + +echo "==> C++ coverage build (Debug, testing=True, instrumented)" +XMSGRID_COVERAGE=1 python build.py \ + --filter='{"build_type":"Debug","testing":true}' + +echo "==> Python wheel build (Release, pybind=True, not instrumented)" +mkdir -p wheelhouse +python build.py \ + --filter='{"build_type":"Release","pybind":true}' \ + --wheel-dir wheelhouse + +echo "==> Locating Conan build folders that wrote .gcda files" +CONAN_HOME="${CONAN_HOME:-$HOME/.conan2}" +mapfile -t GCDA_FILES < <(find "$CONAN_HOME/p" -newer "$MARK" -name '*.gcda' 2>/dev/null || true) + +if [[ ${#GCDA_FILES[@]} -eq 0 ]]; then + echo "error: no .gcda files found under $CONAN_HOME/p; did the C++ build run?" >&2 + exit 1 +fi + +# Walk up from each .gcda to find the directory containing CMakeCache.txt. +declare -A BUILD_FOLDERS=() +for gcda in "${GCDA_FILES[@]}"; do + dir="$(dirname "$gcda")" + while [[ "$dir" != "/" && ! -f "$dir/CMakeCache.txt" ]]; do + dir="$(dirname "$dir")" + done + if [[ -f "$dir/CMakeCache.txt" ]]; then + BUILD_FOLDERS["$dir"]=1 + fi +done + +if [[ ${#BUILD_FOLDERS[@]} -eq 0 ]]; then + echo "error: could not locate any CMakeCache.txt above the discovered .gcda files" >&2 + exit 1 +fi + +echo "==> Found ${#BUILD_FOLDERS[@]} build folder(s):" +for bf in "${!BUILD_FOLDERS[@]}"; do + echo " $bf" +done + +mkdir -p build/coverage-html-cpp build/coverage-html-py + +GCOVR_FILTERS=( + --root "$REPO_ROOT" + --filter "xmsgrid/" + --exclude '.*\.t\.h$' + --exclude 'xmsgrid/python/.*' + --exclude '_package/tests/.*' +) + +echo "==> Aggregating C++ coverage with gcovr" +if [[ ${#BUILD_FOLDERS[@]} -eq 1 ]]; then + bf="${!BUILD_FOLDERS[*]}" + gcovr "${GCOVR_FILTERS[@]}" \ + --xml-pretty --output cov-cpp.xml \ + --html-details "build/coverage-html-cpp/index.html" \ + --print-summary \ + "$bf" +else + # Multiple build folders: emit one JSON per folder, then merge. + JSON_FILES=() + i=0 + for bf in "${!BUILD_FOLDERS[@]}"; do + out="build/coverage-cpp-$i.json" + gcovr "${GCOVR_FILTERS[@]}" --json --output "$out" "$bf" + JSON_FILES+=(--add-tracefile "$out") + i=$((i + 1)) + done + gcovr "${GCOVR_FILTERS[@]}" \ + "${JSON_FILES[@]}" \ + --xml-pretty --output cov-cpp.xml \ + --html-details "build/coverage-html-cpp/index.html" \ + --print-summary +fi + +echo "==> Python coverage run" +PY_VENV="$REPO_ROOT/.coverage-venv" +rm -rf "$PY_VENV" +python -m venv "$PY_VENV" +"$PY_VENV/bin/pip" install --quiet --upgrade pip +"$PY_VENV/bin/pip" install --quiet pytest pytest-cov +WHEEL=$(ls wheelhouse/*.whl | head -n1) +if [[ -z "$WHEEL" ]]; then + echo "error: no wheel found in wheelhouse/" >&2 + exit 1 +fi +"$PY_VENV/bin/pip" install --quiet "$WHEEL" + +"$PY_VENV/bin/pytest" \ + --cov=xms.grid \ + --cov-report=xml:cov-py.xml \ + --cov-report=html:build/coverage-html-py \ + --cov-report=term \ + _package/tests + +echo +echo "==> Coverage reports written:" +echo " cov-cpp.xml Cobertura, C++" +echo " cov-py.xml Cobertura, Python" +echo " build/coverage-html-cpp/ Browsable C++ report" +echo " build/coverage-html-py/ Browsable Python report" diff --git a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md index 4bf57f5b..d23bdc1f 100644 --- a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md +++ b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md @@ -10,10 +10,28 @@ Add unified line-coverage reporting for both the C++ library and Python bindings of `xmsgrid`. Reports are generated locally for developer iteration and in CI for accountability, with results published to Codecov. -The C++ extension is built **once** with coverage instrumentation; both the -ctest and pytest suites run against that single build, and their gcov data is -merged into one C++ report. Python-side coverage is collected separately via -coverage.py. +In Phase 1, the C++ and Python layers are measured independently: + +- **C++ coverage** is collected from a Debug+testing Conan build, instrumented + with `--coverage`, with ctest running the C++ test suite. +- **Python coverage** is collected from a separate pytest run against the + built wheel in a developer-controlled venv, using `pytest-cov`. + +Cross-credit (Python tests contributing to C++ coverage numbers) is **out of +scope for Phase 1** and is the headline feature of Phase 2 once the +orchestration is lifted into xmsconan. Three structural constraints in +xmsconan's generated `conanfile.py` make cross-credit costly to implement +without xmsconan changes: + +1. `BUILD_TESTING` and `IS_PYTHON_BUILD` are mutually exclusive in the + generated CMakeLists (`message(FATAL_ERROR ...)`). +2. `pybind=True` is forced to `build_type=Release` by `configure_options()`, + so there is no Debug pybind build available. +3. The pytest run inside `conanfile.build()` creates a venv with only + `numpy, wheel, pytest` — `pytest-cov` is not installed. + +Phase 1 sidesteps all three by treating the suites as independent. Phase 2 +addresses them in xmsconan. ## Non-Goals @@ -28,26 +46,34 @@ coverage.py. ## Architecture ``` -┌─ Coverage Build (Linux, GCC, Debug) ─────────────────────────┐ -│ cmake -DXMSGRID_ENABLE_COVERAGE=ON → --coverage flags │ -│ builds: libxmsgrid (instrumented) │ -│ C++ test runner (instrumented) │ -│ Python extension _xmsgrid (instrumented) │ +┌─ C++ Coverage Build (Linux, GCC, Debug, testing=True) ───────┐ +│ XMSGRID_COVERAGE=1 → CMake adds --coverage to all targets │ +│ python build.py --filter='{"build_type":"Debug", │ +│ "testing":true}' │ +│ conanfile.run_cxx_tests() → ctest → *.gcda in conan cache │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ gcovr scans conan build │ + │ folder → cobertura-cpp.xml │ + └─────────────────────────────┘ + +┌─ Python Coverage Run (Linux, against built wheel) ───────────┐ +│ python build.py --filter='{"build_type":"Release", │ +│ "pybind":true}' │ +│ --wheel-dir wheelhouse │ +│ Then in a clean venv: │ +│ pip install wheelhouse/*.whl pytest pytest-cov │ +│ pytest --cov=xms.grid --cov-report=xml _package/tests │ └──────────────────────────────────────────────────────────────┘ - │ │ - ▼ ▼ - ┌──────────────────┐ ┌─────────────────────┐ - │ ctest │ │ pytest │ - │ → *.gcda files │ │ → *.gcda (C++) │ - │ (C++ tests) │ │ → .coverage (py) │ - └──────────────────┘ └─────────────────────┘ - │ │ - └─────────────┬──────────────────┘ + │ ▼ ┌─────────────────────────────┐ - │ gcovr → cobertura-cpp.xml │ - │ coverage → cobertura-py.xml │ + │ coverage.py → cov-py.xml │ └─────────────────────────────┘ + + │ ▼ ┌─────────────────────────────┐ │ Codecov upload │ @@ -56,70 +82,84 @@ coverage.py. └─────────────────────────────┘ ``` -The crucial property: pybind11-built objects compiled with `--coverage` emit -`.gcda` files when their code paths run, regardless of whether the caller is -C++ or Python. So pytest exercising the bindings credits the underlying C++ -implementation in the C++ report. +The C++ build is instrumented; the Python wheel build is **not**. Python +coverage measures Python-level wrappers in `_package/xms/grid/` only. +Cross-instrumentation is Phase 2. ## Components ### 1. CMake option (in `build.toml` `extra_cmake_text`) ```cmake -option(XMSGRID_ENABLE_COVERAGE "Build with gcov instrumentation" OFF) -if(XMSGRID_ENABLE_COVERAGE) +if(DEFINED ENV{XMSGRID_COVERAGE} AND NOT "$ENV{XMSGRID_COVERAGE}" STREQUAL "") if(MSVC) message(FATAL_ERROR "Coverage build is GCC/Clang only") endif() + message(STATUS "Coverage instrumentation enabled (XMSGRID_COVERAGE)") add_compile_options(--coverage -O0 -g) add_link_options(--coverage) endif() ``` -The flag is added to the existing `extra_cmake_text` block — no xmsconan -generator change. Off by default, so non-coverage builds are untouched. +Gated on an environment variable instead of a CMake cache variable so the +coverage script can flip it without needing to pass `--cmake-args` through +xmsconan's `build.py`. No xmsconan generator change required. The variable +flows naturally through Conan's `tools.env.virtualenv` and into the CMake +configure step. ### 2. `dev/coverage.sh` POSIX shell script (Linux + macOS). Single entry point. Steps: -1. Run the Conan build with the coverage flag: +1. Drop a marker file (`.coverage-mark`) — used to identify build folders + that this run wrote `.gcda` files into. +2. Run the C++ coverage build: + ``` + XMSGRID_COVERAGE=1 python build.py \ + --filter='{"build_type":"Debug","testing":true}' + ``` + The conanfile's `run_cxx_tests()` runs ctest as part of `build()`, so + this single invocation builds, runs, and emits `.gcda` files into + `~/.conan2/p//b/build/...`. +3. Run the wheel build: ``` python build.py \ - --filter='{"build_type":"Debug"}' \ - --cmake-args='-DXMSGRID_ENABLE_COVERAGE=ON' \ + --filter='{"build_type":"Release","pybind":true}' \ --wheel-dir wheelhouse ``` -2. Run the C++ test suite via `ctest` against the instrumented build (the - conan recipe already wires this). -3. Install the freshly-built wheel into a throwaway venv: + No coverage env var here — the wheel is for Python coverage only and + does not need instrumentation in Phase 1. +4. Discover C++ build folders that received fresh `.gcda` files: ``` - python -m venv .coverage-venv - .coverage-venv/bin/pip install wheelhouse/*.whl pytest pytest-cov + find ~/.conan2/p -newer .coverage-mark -name '*.gcda' + ``` + Walk up from each match to the directory containing `CMakeCache.txt`; + that's the build folder gcovr needs to scan. Deduplicate the set. +5. Run gcovr against the source root with each discovered build folder: ``` -4. Run the Python tests with coverage: + gcovr --root . \ + --filter 'xmsgrid/' \ + --exclude '.*\.t\.h$' \ + --exclude 'xmsgrid/python/.*' \ + --exclude '_package/tests/.*' \ + --xml cov-cpp.xml \ + --html-details build/coverage-html-cpp/index.html \ + + ``` + When multiple build folders are found (rare in Phase 1, but possible + if shards are used), emit a JSON intermediate per folder via + `--json-summary`, then merge with `gcovr --add-tracefile a.json -a b.json`. +6. Run Python tests with coverage in a clean venv: ``` + python -m venv .coverage-venv + .coverage-venv/bin/pip install wheelhouse/*.whl pytest pytest-cov .coverage-venv/bin/pytest \ --cov=xms.grid \ --cov-report=xml:cov-py.xml \ --cov-report=html:build/coverage-html-py \ _package/tests ``` -5. Aggregate C++ gcov data: - ``` - gcovr --root . \ - --filter 'xmsgrid/' \ - --exclude '.*\.t\.h$' \ - --exclude 'xmsgrid/python/.*' \ - --exclude '_package/tests/.*' \ - --xml cov-cpp.xml \ - --html-details build/coverage-html-cpp/index.html - ``` -6. Print a summary line, e.g. `Coverage: C++ 78.3%, Python 64.1%`. - -The wheel installed in step 3 must be the exact one produced in step 1 — -gcov data files reference build-tree absolute paths, so any rebuild between -the two will desync the `.gcda` / `.gcno` pair. +7. Print a summary line, e.g. `Coverage: C++ 78.3%, Python 64.1%`. A `dev/coverage.bat` is **not** added in Phase 1; Windows coverage is out of scope. From e9a7dbbdab6065b91d777cd86b2fd5d93e1bcbf3 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:08:26 -0600 Subject: [PATCH 03/11] Swap coverage gating from Codecov to in-CI thresholds Drops codecov.yml and the codecov-action upload steps. Adds --fail-under-line / --cov-fail-under flags driven by CPP_COVERAGE_THRESHOLD and PY_COVERAGE_THRESHOLD env vars (default 0), set explicitly in Coverage.yaml. CI fails when either layer falls below its configured threshold. HTML and Cobertura XML reports still upload as workflow artifacts on every run (with if: always()) so reviewers can inspect numbers even on threshold failures. The XMLs remain in standard format, so adopting a reporting service later is a small additional step. Spec updated: Phase 1 explicitly defers the reporting-service decision. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/Coverage.yaml | 31 +++--- README.md | 9 +- codecov.yml | 28 ----- dev/coverage.sh | 42 +++++-- .../2026-05-08-coverage-reporting-design.md | 103 +++++++++--------- 5 files changed, 100 insertions(+), 113 deletions(-) delete mode 100644 codecov.yml diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml index c1a26735..552f1af9 100644 --- a/.github/workflows/Coverage.yaml +++ b/.github/workflows/Coverage.yaml @@ -7,7 +7,6 @@ # AQUAPI_USERNAME_SECRET - devpi username for xmsconan install # AQUAPI_PASSWORD_SECRET - devpi password for xmsconan install # AQUAPI_URL_DEV - devpi index URL for xmsconan install -# CODECOV_TOKEN - Codecov upload token name: Coverage @@ -39,6 +38,10 @@ jobs: PYTHON_TARGET_VERSION: '3.13' RELEASE_PYTHON: 'False' CTEST_PARALLEL_LEVEL: '8' + # Coverage thresholds. Bump these once a baseline is established. + # The job fails if either falls below its threshold. + CPP_COVERAGE_THRESHOLD: '0' + PY_COVERAGE_THRESHOLD: '0' steps: - name: Checkout Source @@ -57,22 +60,6 @@ jobs: run: dev/coverage.sh shell: bash - - name: Upload C++ coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: cov-cpp.xml - flags: cpp - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - - - name: Upload Python coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: cov-py.xml - flags: python - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - - name: Upload HTML reports as artifact uses: actions/upload-artifact@v4 if: always() @@ -82,3 +69,13 @@ jobs: build/coverage-html-cpp build/coverage-html-py retention-days: 14 + + - name: Upload Cobertura XML as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-xml + path: | + cov-cpp.xml + cov-py.xml + retention-days: 14 diff --git a/README.md b/README.md index cf2f1801..8594fce6 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,15 @@ Coverage runs on Linux/macOS via `dev/coverage.sh`. The script runs an instrumented Debug build with the C++ test suite, then a release wheel build with pytest in a clean venv. Outputs: -- `cov-cpp.xml` / `cov-py.xml` — Cobertura XML for upload to Codecov. +- `cov-cpp.xml` / `cov-py.xml` — Cobertura XML. - `build/coverage-html-cpp/` and `build/coverage-html-py/` — browsable HTML. Prerequisites: `gcovr` (`pip install gcovr`) and a working Conan toolchain (see Building above). Phase 1 is GCC/Clang-only; Windows MSVC coverage is not supported. -CI publishes both flags to Codecov on every push and pull request. Codecov -is informational only — it will not block PRs. +CI runs the same script and uploads the HTML and XML reports as workflow +artifacts. Coverage is gated by simple percentage thresholds set in +`.github/workflows/Coverage.yaml` (`CPP_COVERAGE_THRESHOLD` and +`PY_COVERAGE_THRESHOLD`); the job fails if either layer falls below. +Initial values are `0` until a baseline is established. diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index fb0fbadf..00000000 --- a/codecov.yml +++ /dev/null @@ -1,28 +0,0 @@ -coverage: - status: - project: - default: - informational: true - patch: - default: - informational: true - -flags: - cpp: - paths: - - xmsgrid/ - carryforward: true - python: - paths: - - _package/xms/ - carryforward: true - -ignore: - - "_package/tests/" - - "xmsgrid/**/*.t.h" - - "xmsgrid/python/" - -comment: - layout: "header, diff, flags, files" - behavior: default - require_changes: false diff --git a/dev/coverage.sh b/dev/coverage.sh index 1b2aa700..366defe5 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -96,14 +96,20 @@ GCOVR_FILTERS=( --exclude '_package/tests/.*' ) -echo "==> Aggregating C++ coverage with gcovr" +# Thresholds. Default 0 means "do not fail". Override via env var to gate. +CPP_COVERAGE_THRESHOLD="${CPP_COVERAGE_THRESHOLD:-0}" +PY_COVERAGE_THRESHOLD="${PY_COVERAGE_THRESHOLD:-0}" + +echo "==> Aggregating C++ coverage with gcovr (threshold: $CPP_COVERAGE_THRESHOLD%)" +GCOVR_OUTPUT=( + --xml-pretty --output cov-cpp.xml + --html-details "build/coverage-html-cpp/index.html" + --print-summary + --fail-under-line "$CPP_COVERAGE_THRESHOLD" +) if [[ ${#BUILD_FOLDERS[@]} -eq 1 ]]; then bf="${!BUILD_FOLDERS[*]}" - gcovr "${GCOVR_FILTERS[@]}" \ - --xml-pretty --output cov-cpp.xml \ - --html-details "build/coverage-html-cpp/index.html" \ - --print-summary \ - "$bf" + gcovr "${GCOVR_FILTERS[@]}" "${GCOVR_OUTPUT[@]}" "$bf" else # Multiple build folders: emit one JSON per folder, then merge. JSON_FILES=() @@ -114,14 +120,10 @@ else JSON_FILES+=(--add-tracefile "$out") i=$((i + 1)) done - gcovr "${GCOVR_FILTERS[@]}" \ - "${JSON_FILES[@]}" \ - --xml-pretty --output cov-cpp.xml \ - --html-details "build/coverage-html-cpp/index.html" \ - --print-summary + gcovr "${GCOVR_FILTERS[@]}" "${JSON_FILES[@]}" "${GCOVR_OUTPUT[@]}" fi -echo "==> Python coverage run" +echo "==> Python coverage run (threshold: $PY_COVERAGE_THRESHOLD%)" PY_VENV="$REPO_ROOT/.coverage-venv" rm -rf "$PY_VENV" python -m venv "$PY_VENV" @@ -139,8 +141,24 @@ fi --cov-report=xml:cov-py.xml \ --cov-report=html:build/coverage-html-py \ --cov-report=term \ + --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ _package/tests +# Optional: append summary to GitHub Actions step summary. +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "## Coverage Summary" + echo + echo "| Layer | Threshold |" + echo "|-------|-----------|" + echo "| C++ | $CPP_COVERAGE_THRESHOLD% |" + echo "| Python | $PY_COVERAGE_THRESHOLD% |" + echo + echo "See \`gcovr\` and \`pytest --cov\` output above for actual percentages." + echo "Browseable HTML reports are uploaded as the \`coverage-html\` artifact." + } >> "$GITHUB_STEP_SUMMARY" +fi + echo echo "==> Coverage reports written:" echo " cov-cpp.xml Cobertura, C++" diff --git a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md index d23bdc1f..ec8b7031 100644 --- a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md +++ b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md @@ -8,7 +8,9 @@ Add unified line-coverage reporting for both the C++ library and Python bindings of `xmsgrid`. Reports are generated locally for developer iteration -and in CI for accountability, with results published to Codecov. +and in CI for accountability. CI gates merges with simple percentage +thresholds rather than a third-party reporting service. Adopting Codecov or +similar is an open follow-up decision after a baseline is established. In Phase 1, the C++ and Python layers are measured independently: @@ -76,9 +78,9 @@ addresses them in xmsconan. │ ▼ ┌─────────────────────────────┐ - │ Codecov upload │ - │ flag: cpp │ - │ flag: python │ + │ Threshold check │ + │ gcovr --fail-under-line │ + │ pytest --cov-fail-under │ └─────────────────────────────┘ ``` @@ -86,6 +88,11 @@ The C++ build is instrumented; the Python wheel build is **not**. Python coverage measures Python-level wrappers in `_package/xms/grid/` only. Cross-instrumentation is Phase 2. +The reports (Cobertura XML + browsable HTML) upload as workflow artifacts +on every CI run regardless of pass/fail, so reviewers can always inspect +the numbers. If the team later decides to adopt Codecov or a similar +service, the cobertura XMLs are already in the right format to drop in. + ## Components ### 1. CMake option (in `build.toml` `extra_cmake_text`) @@ -149,7 +156,8 @@ POSIX shell script (Linux + macOS). Single entry point. Steps: When multiple build folders are found (rare in Phase 1, but possible if shards are used), emit a JSON intermediate per folder via `--json-summary`, then merge with `gcovr --add-tracefile a.json -a b.json`. -6. Run Python tests with coverage in a clean venv: +6. Run Python tests with coverage in a clean venv, gated by the Python + threshold: ``` python -m venv .coverage-venv .coverage-venv/bin/pip install wheelhouse/*.whl pytest pytest-cov @@ -157,9 +165,19 @@ POSIX shell script (Linux + macOS). Single entry point. Steps: --cov=xms.grid \ --cov-report=xml:cov-py.xml \ --cov-report=html:build/coverage-html-py \ + --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ _package/tests ``` -7. Print a summary line, e.g. `Coverage: C++ 78.3%, Python 64.1%`. +7. The C++ gcovr invocation in step 5 passes + `--fail-under-line "$CPP_COVERAGE_THRESHOLD"` so it exits non-zero when + the C++ percentage falls below. +8. When running under GitHub Actions (`$GITHUB_STEP_SUMMARY` set), append + a short markdown summary noting the configured thresholds. The actual + percentages appear in the gcovr / pytest output above the summary. + +Thresholds default to `0` (no gating) via env vars +`CPP_COVERAGE_THRESHOLD` and `PY_COVERAGE_THRESHOLD`. CI sets them +explicitly in `Coverage.yaml`; locally they can be overridden via env. A `dev/coverage.bat` is **not** added in Phase 1; Windows coverage is out of scope. @@ -170,65 +188,44 @@ Hand-maintained, separate from the xmsconan-generated `XmsGrid-CI.yaml`. Single job, Linux, GCC, Debug: - Triggers: `push`, `pull_request`. -- Checks out source, installs xmsconan + Python deps, sets up Conan login. -- Runs `dev/coverage.sh`. -- Uploads `cov-cpp.xml` to Codecov with flag `cpp`. -- Uploads `cov-py.xml` to Codecov with flag `python`. -- Uploads `build/coverage-html-cpp` and `build/coverage-html-py` as - workflow artifacts so a developer can download and browse them. - -The job uses the same Conan secrets as the existing CI workflow and the -new `CODECOV_TOKEN` repo secret. - -### 4. `codecov.yml` - -```yaml -coverage: - status: - project: - default: - informational: true - patch: - default: - informational: true -flags: - cpp: - paths: - - xmsgrid/ - carryforward: true - python: - paths: - - _package/xms/ - carryforward: true -ignore: - - "_package/tests/" - - "xmsgrid/**/*.t.h" - - "xmsgrid/python/" -``` +- Checks out source, installs xmsconan + gcovr + Python deps, sets up Conan + login. +- Sets `CPP_COVERAGE_THRESHOLD` and `PY_COVERAGE_THRESHOLD` (initially `0`, + bumped after the first run establishes a baseline). +- Runs `dev/coverage.sh`. The script's gcovr / pytest steps fail the job + when either threshold is missed. +- Uploads `cov-cpp.xml` and `cov-py.xml` as the `coverage-xml` artifact and + the HTML reports as the `coverage-html` artifact, with `if: always()` so + they are available even on threshold failure. -Both project and patch status are informational — Codecov will comment on -PRs with the numbers but won't fail the check. Tightening is a future -decision once we have a baseline. +The job uses the same Conan / devpi secrets as the existing CI workflow. +No reporting-service token is required. -### 5. README update +### 4. README update Short `## Coverage` section: how to run `dev/coverage.sh`, where the HTML -lands, link to the Codecov project page. +lands, how thresholds are configured. ## Repo Setup (one-time, manual) -- Add the `xmsgrid` repo to Codecov via the GitHub app. -- Add `CODECOV_TOKEN` as a repository secret in GitHub settings. +None for Phase 1 beyond the secrets the existing workflow already requires. +The first CI run establishes a baseline; bump the threshold env values in +`Coverage.yaml` afterwards to enforce no-regression. + +## Reporting Service Decision (deferred) -These are operator actions, not code changes; they should be documented in -the implementation plan as prerequisites for the Coverage workflow's first -green run. +Whether to adopt Codecov or a similar service — for diff coverage, +trend tracking, or PR comments — is intentionally deferred to a follow-up +decision after the team sees a few weeks of threshold-gated runs and can +judge whether richer signal is worth the integration. The cobertura XMLs +emitted by Phase 1 are already in the standard format any of these +services accept, so adopting one later is mechanical. ## Phase 2 — Lift to xmsconan After Phase 1 ships and stabilizes (a few real PRs through the new -workflow, no broken Codecov runs for a week), lift the generic pieces -into xmsconan in a follow-up PR. +workflow, thresholds set above zero, no flaky failures for a week), lift +the generic pieces into xmsconan in a follow-up PR. **Lifted into xmsconan:** From d7ebd2d96ad273b944d1ae8bbece6ed278e482f0 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:24:05 -0600 Subject: [PATCH 04/11] Gitignore xmsconan-generated .flake8 and pytest.ini Both are rendered by xmsconan_gen from templates and should not be tracked, matching the treatment of CMakeLists.txt, conanfile.py, etc. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index dc2629e7..911be830 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ conanfile.py build.py xms_conan2_file.py _package/pyproject.toml +.flake8 +pytest.ini # ── Build output ─────────────────────────────── build/ From a05ff63846e6cef108199efba84b479dc5f35726 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:26:52 -0600 Subject: [PATCH 05/11] Fix coverage CI: gcovr root + pip index for xmscore C++: gcovr was given the repo root as --root, but Conan compiles sources out of ~/.conan2/p//b/src/, so the repo-relative xmsgrid/ filter matched nothing ("All coverage data is filtered out"). Discover the matching source folder per build folder (cmake_layout puts it at the sibling 'src' directory two parents up from the build folder) and pass it as --root. Python: the wheel declares an xmscore>=7.0.8 runtime dep that lives on Aquaveo devpi, not PyPI. Add --extra-index-url to the wheel pip install so pip can resolve it. Honors COVERAGE_PIP_INDEX for override. Co-Authored-By: Claude Opus 4.7 --- dev/coverage.sh | 68 +++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/dev/coverage.sh b/dev/coverage.sh index 366defe5..2dfe1c51 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -64,63 +64,78 @@ if [[ ${#GCDA_FILES[@]} -eq 0 ]]; then exit 1 fi -# Walk up from each .gcda to find the directory containing CMakeCache.txt. -declare -A BUILD_FOLDERS=() +# Walk up from each .gcda to find the directory containing CMakeCache.txt; +# pair each with its source folder. cmake_layout puts source at /b/src +# and build at /b/build/, so source is two parents up + /src. +declare -A BUILD_TO_SOURCE=() for gcda in "${GCDA_FILES[@]}"; do dir="$(dirname "$gcda")" while [[ "$dir" != "/" && ! -f "$dir/CMakeCache.txt" ]]; do dir="$(dirname "$dir")" done - if [[ -f "$dir/CMakeCache.txt" ]]; then - BUILD_FOLDERS["$dir"]=1 + if [[ ! -f "$dir/CMakeCache.txt" ]]; then + continue fi + src_dir="$(cd "$dir/../.." && pwd)/src" + if [[ ! -d "$src_dir/xmsgrid" ]]; then + echo "error: expected source dir at $src_dir but xmsgrid/ not found" >&2 + exit 1 + fi + BUILD_TO_SOURCE["$dir"]="$src_dir" done -if [[ ${#BUILD_FOLDERS[@]} -eq 0 ]]; then +if [[ ${#BUILD_TO_SOURCE[@]} -eq 0 ]]; then echo "error: could not locate any CMakeCache.txt above the discovered .gcda files" >&2 exit 1 fi -echo "==> Found ${#BUILD_FOLDERS[@]} build folder(s):" -for bf in "${!BUILD_FOLDERS[@]}"; do - echo " $bf" +echo "==> Found ${#BUILD_TO_SOURCE[@]} build folder(s):" +for bf in "${!BUILD_TO_SOURCE[@]}"; do + echo " build: $bf" + echo " source: ${BUILD_TO_SOURCE[$bf]}" done mkdir -p build/coverage-html-cpp build/coverage-html-py -GCOVR_FILTERS=( - --root "$REPO_ROOT" - --filter "xmsgrid/" - --exclude '.*\.t\.h$' - --exclude 'xmsgrid/python/.*' - --exclude '_package/tests/.*' -) - # Thresholds. Default 0 means "do not fail". Override via env var to gate. CPP_COVERAGE_THRESHOLD="${CPP_COVERAGE_THRESHOLD:-0}" PY_COVERAGE_THRESHOLD="${PY_COVERAGE_THRESHOLD:-0}" -echo "==> Aggregating C++ coverage with gcovr (threshold: $CPP_COVERAGE_THRESHOLD%)" +GCOVR_COMMON=( + --filter 'xmsgrid/' + --exclude '.*\.t\.h$' + --exclude 'xmsgrid/python/.*' + --exclude '_package/tests/.*' +) GCOVR_OUTPUT=( --xml-pretty --output cov-cpp.xml --html-details "build/coverage-html-cpp/index.html" --print-summary --fail-under-line "$CPP_COVERAGE_THRESHOLD" ) -if [[ ${#BUILD_FOLDERS[@]} -eq 1 ]]; then - bf="${!BUILD_FOLDERS[*]}" - gcovr "${GCOVR_FILTERS[@]}" "${GCOVR_OUTPUT[@]}" "$bf" + +echo "==> Aggregating C++ coverage with gcovr (threshold: $CPP_COVERAGE_THRESHOLD%)" +# gcovr's --root must be the source folder; the build folder argument tells +# it where to find .gcda files. With one build folder we emit directly; +# with multiple we collect JSON intermediates then merge. +if [[ ${#BUILD_TO_SOURCE[@]} -eq 1 ]]; then + bf="${!BUILD_TO_SOURCE[*]}" + src="${BUILD_TO_SOURCE[$bf]}" + gcovr --root "$src" "${GCOVR_COMMON[@]}" "${GCOVR_OUTPUT[@]}" "$bf" else - # Multiple build folders: emit one JSON per folder, then merge. JSON_FILES=() i=0 - for bf in "${!BUILD_FOLDERS[@]}"; do + for bf in "${!BUILD_TO_SOURCE[@]}"; do + src="${BUILD_TO_SOURCE[$bf]}" out="build/coverage-cpp-$i.json" - gcovr "${GCOVR_FILTERS[@]}" --json --output "$out" "$bf" + gcovr --root "$src" "${GCOVR_COMMON[@]}" --json --output "$out" "$bf" JSON_FILES+=(--add-tracefile "$out") i=$((i + 1)) done - gcovr "${GCOVR_FILTERS[@]}" "${JSON_FILES[@]}" "${GCOVR_OUTPUT[@]}" + # Merge: --root here only affects how merged paths are rendered, not + # which gcda files are read. Use the first source folder as the root. + first_src="${BUILD_TO_SOURCE[$(printf '%s\n' "${!BUILD_TO_SOURCE[@]}" | head -n1)]}" + gcovr --root "$first_src" "${GCOVR_COMMON[@]}" "${JSON_FILES[@]}" "${GCOVR_OUTPUT[@]}" fi echo "==> Python coverage run (threshold: $PY_COVERAGE_THRESHOLD%)" @@ -134,7 +149,10 @@ if [[ -z "$WHEEL" ]]; then echo "error: no wheel found in wheelhouse/" >&2 exit 1 fi -"$PY_VENV/bin/pip" install --quiet "$WHEEL" +# xmsgrid wheels declare an xmscore dependency that lives on Aquaveo devpi, +# not PyPI. Pass the public devpi index so pip can resolve it. +DEVPI_INDEX="${COVERAGE_PIP_INDEX:-https://public.aquapi.aquaveo.com/aquaveo/dev/+simple}" +"$PY_VENV/bin/pip" install --quiet --extra-index-url "$DEVPI_INDEX" "$WHEEL" "$PY_VENV/bin/pytest" \ --cov=xms.grid \ From ac99453d07f56b4abad674161655471be1ffc2fb Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:31:57 -0600 Subject: [PATCH 06/11] Discover source folder by scanning, not guessing cmake_layout in this project puts source at /b/xmsgrid/, not /b/src/xmsgrid/ as I assumed. Replace the hardcoded ../../src path with an actual scan: walk up from the build folder looking for the first ancestor that contains an xmsgrid/ directory. Co-Authored-By: Claude Opus 4.7 --- dev/coverage.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dev/coverage.sh b/dev/coverage.sh index 2dfe1c51..03bbcf7d 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -64,9 +64,9 @@ if [[ ${#GCDA_FILES[@]} -eq 0 ]]; then exit 1 fi -# Walk up from each .gcda to find the directory containing CMakeCache.txt; -# pair each with its source folder. cmake_layout puts source at /b/src -# and build at /b/build/, so source is two parents up + /src. +# Walk up from each .gcda to find the directory containing CMakeCache.txt +# (the build folder), then walk further up looking for the conan source +# folder — the first parent that contains a sibling 'xmsgrid' directory. declare -A BUILD_TO_SOURCE=() for gcda in "${GCDA_FILES[@]}"; do dir="$(dirname "$gcda")" @@ -76,9 +76,12 @@ for gcda in "${GCDA_FILES[@]}"; do if [[ ! -f "$dir/CMakeCache.txt" ]]; then continue fi - src_dir="$(cd "$dir/../.." && pwd)/src" + src_dir="$(dirname "$dir")" + while [[ "$src_dir" != "/" && ! -d "$src_dir/xmsgrid" ]]; do + src_dir="$(dirname "$src_dir")" + done if [[ ! -d "$src_dir/xmsgrid" ]]; then - echo "error: expected source dir at $src_dir but xmsgrid/ not found" >&2 + echo "error: could not locate a parent of $dir containing xmsgrid/" >&2 exit 1 fi BUILD_TO_SOURCE["$dir"]="$src_dir" From 28985c978a5642ad866250e90bf7164b6be1edb6 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:40:05 -0600 Subject: [PATCH 07/11] Fix gcovr filter regex and pytest sys.path shadowing C++: gcovr's path-prefix filter resolution against .gcno-recorded absolute source paths was producing 'all data filtered out' even with --root set correctly. Use an explicit anywhere-in-path regex ('.*/xmsgrid/.*') to bypass the prefix-matching ambiguity. Python: pytest's auto-rootdir discovery was finding the in-tree _package/xms/grid/ directory and putting it on sys.path, shadowing the venv's installed wheel and producing 'cannot import _xmsgrid' errors. Copy tests to build/coverage-tests/ and run pytest from there, mirroring what the conanfile's own python test runner does. Co-Authored-By: Claude Opus 4.7 --- dev/coverage.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/dev/coverage.sh b/dev/coverage.sh index 03bbcf7d..f6ad3a80 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -105,10 +105,10 @@ CPP_COVERAGE_THRESHOLD="${CPP_COVERAGE_THRESHOLD:-0}" PY_COVERAGE_THRESHOLD="${PY_COVERAGE_THRESHOLD:-0}" GCOVR_COMMON=( - --filter 'xmsgrid/' + --filter '.*/xmsgrid/.*' --exclude '.*\.t\.h$' - --exclude 'xmsgrid/python/.*' - --exclude '_package/tests/.*' + --exclude '.*/xmsgrid/python/.*' + --exclude '.*/_package/tests/.*' ) GCOVR_OUTPUT=( --xml-pretty --output cov-cpp.xml @@ -157,13 +157,25 @@ fi DEVPI_INDEX="${COVERAGE_PIP_INDEX:-https://public.aquapi.aquaveo.com/aquaveo/dev/+simple}" "$PY_VENV/bin/pip" install --quiet --extra-index-url "$DEVPI_INDEX" "$WHEEL" -"$PY_VENV/bin/pytest" \ - --cov=xms.grid \ - --cov-report=xml:cov-py.xml \ - --cov-report=html:build/coverage-html-py \ - --cov-report=term \ - --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ - _package/tests +# Copy tests outside the source tree before running pytest. Otherwise +# pytest's rootdir/sys.path discovery picks up the local _package/xms/grid/ +# source directory (which has no compiled _xmsgrid extension) and shadows +# the installed wheel, producing import errors. The conanfile's own python +# test runner does the same thing. +TESTS_COPY="$REPO_ROOT/build/coverage-tests" +rm -rf "$TESTS_COPY" +cp -r _package/tests "$TESTS_COPY" + +( + cd "$TESTS_COPY" + "$PY_VENV/bin/pytest" \ + --cov=xms.grid \ + --cov-report=xml:"$REPO_ROOT/cov-py.xml" \ + --cov-report=html:"$REPO_ROOT/build/coverage-html-py" \ + --cov-report=term \ + --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ + . +) # Optional: append summary to GitHub Actions step summary. if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then From 98e1843d0cc3513e7c0aaca590f6fe0fb58c2911 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 10:58:23 -0600 Subject: [PATCH 08/11] Show actuals in summary table and pin thresholds to baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage summary table now reports threshold, actual, and pass/fail status per layer. Switch from gcovr's --fail-under-line and pytest-cov's --cov-fail-under (which would short-circuit before the summary block) to parsing each tool's JSON summary, doing the threshold check ourselves, and exiting non-zero at the end if any layer is below. Pin CPP_COVERAGE_THRESHOLD=74.7 and PY_COVERAGE_THRESHOLD=83 in Coverage.yaml — the baseline numbers from the first green run. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/Coverage.yaml | 8 +++--- dev/coverage.sh | 46 ++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml index 552f1af9..89740541 100644 --- a/.github/workflows/Coverage.yaml +++ b/.github/workflows/Coverage.yaml @@ -38,10 +38,10 @@ jobs: PYTHON_TARGET_VERSION: '3.13' RELEASE_PYTHON: 'False' CTEST_PARALLEL_LEVEL: '8' - # Coverage thresholds. Bump these once a baseline is established. - # The job fails if either falls below its threshold. - CPP_COVERAGE_THRESHOLD: '0' - PY_COVERAGE_THRESHOLD: '0' + # Coverage thresholds. Pinned to current baseline; bump as coverage + # improves. The job fails if either layer falls below its threshold. + CPP_COVERAGE_THRESHOLD: '74.7' + PY_COVERAGE_THRESHOLD: '83' steps: - name: Checkout Source diff --git a/dev/coverage.sh b/dev/coverage.sh index f6ad3a80..23612a7b 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -113,14 +113,16 @@ GCOVR_COMMON=( GCOVR_OUTPUT=( --xml-pretty --output cov-cpp.xml --html-details "build/coverage-html-cpp/index.html" + --json-summary-pretty --json-summary "build/cov-cpp-summary.json" --print-summary - --fail-under-line "$CPP_COVERAGE_THRESHOLD" ) echo "==> Aggregating C++ coverage with gcovr (threshold: $CPP_COVERAGE_THRESHOLD%)" # gcovr's --root must be the source folder; the build folder argument tells # it where to find .gcda files. With one build folder we emit directly; -# with multiple we collect JSON intermediates then merge. +# with multiple we collect JSON intermediates then merge. Threshold check +# is performed below from the json-summary so the summary table can show +# the actual percentages even when a layer fails. if [[ ${#BUILD_TO_SOURCE[@]} -eq 1 ]]; then bf="${!BUILD_TO_SOURCE[*]}" src="${BUILD_TO_SOURCE[$bf]}" @@ -141,6 +143,8 @@ else gcovr --root "$first_src" "${GCOVR_COMMON[@]}" "${JSON_FILES[@]}" "${GCOVR_OUTPUT[@]}" fi +CPP_ACTUAL=$(python3 -c "import json; d=json.load(open('build/cov-cpp-summary.json')); print(d.get('line_percent', d.get('lines',{}).get('percent', 0)))") + echo "==> Python coverage run (threshold: $PY_COVERAGE_THRESHOLD%)" PY_VENV="$REPO_ROOT/.coverage-venv" rm -rf "$PY_VENV" @@ -172,20 +176,42 @@ cp -r _package/tests "$TESTS_COPY" --cov=xms.grid \ --cov-report=xml:"$REPO_ROOT/cov-py.xml" \ --cov-report=html:"$REPO_ROOT/build/coverage-html-py" \ + --cov-report=json:"$REPO_ROOT/build/cov-py-summary.json" \ --cov-report=term \ - --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ . ) +PY_ACTUAL=$(python3 -c "import json; print(json.load(open('build/cov-py-summary.json'))['totals']['percent_covered'])") + +# Threshold checks. Both run unconditionally so the summary always reflects +# real numbers; we set FAILED at the end if either layer is below. +FAILED=0 +cpp_pass=$(python3 -c "import sys; sys.exit(0 if float('$CPP_ACTUAL') >= float('$CPP_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +py_pass=$(python3 -c "import sys; sys.exit(0 if float('$PY_ACTUAL') >= float('$PY_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +[[ "$cpp_pass" == 0 ]] && FAILED=1 +[[ "$py_pass" == 0 ]] && FAILED=1 + +cpp_status=$([[ "$cpp_pass" == 1 ]] && echo "pass" || echo "FAIL") +py_status=$([[ "$py_pass" == 1 ]] && echo "pass" || echo "FAIL") + +# Round actuals to one decimal for display. +CPP_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$CPP_ACTUAL\"):.1f}')") +PY_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$PY_ACTUAL\"):.1f}')") + +echo +echo "==> Threshold check" +echo " C++: actual ${CPP_ACTUAL_DISP}% threshold ${CPP_COVERAGE_THRESHOLD}% [$cpp_status]" +echo " Python: actual ${PY_ACTUAL_DISP}% threshold ${PY_COVERAGE_THRESHOLD}% [$py_status]" + # Optional: append summary to GitHub Actions step summary. if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Coverage Summary" echo - echo "| Layer | Threshold |" - echo "|-------|-----------|" - echo "| C++ | $CPP_COVERAGE_THRESHOLD% |" - echo "| Python | $PY_COVERAGE_THRESHOLD% |" + echo "| Layer | Threshold | Actual | Status |" + echo "|--------|-----------|--------|--------|" + echo "| C++ | ${CPP_COVERAGE_THRESHOLD}% | ${CPP_ACTUAL_DISP}% | $cpp_status |" + echo "| Python | ${PY_COVERAGE_THRESHOLD}% | ${PY_ACTUAL_DISP}% | $py_status |" echo echo "See \`gcovr\` and \`pytest --cov\` output above for actual percentages." echo "Browseable HTML reports are uploaded as the \`coverage-html\` artifact." @@ -198,3 +224,9 @@ echo " cov-cpp.xml Cobertura, C++" echo " cov-py.xml Cobertura, Python" echo " build/coverage-html-cpp/ Browsable C++ report" echo " build/coverage-html-py/ Browsable Python report" + +if [[ "$FAILED" -ne 0 ]]; then + echo + echo "error: coverage below threshold for one or more layers" >&2 + exit 1 +fi From eaa9fef11be4843398a1acca49b09de04720610e Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 11:04:08 -0600 Subject: [PATCH 09/11] Pin Python coverage threshold to actual 82.9 (was 83) Previous baseline read 83% from the rounded log line; the precise actual is 82.9. Adjusts the threshold so the very next run passes against the unchanged code. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/Coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml index 89740541..79ff7ed4 100644 --- a/.github/workflows/Coverage.yaml +++ b/.github/workflows/Coverage.yaml @@ -41,7 +41,7 @@ jobs: # Coverage thresholds. Pinned to current baseline; bump as coverage # improves. The job fails if either layer falls below its threshold. CPP_COVERAGE_THRESHOLD: '74.7' - PY_COVERAGE_THRESHOLD: '83' + PY_COVERAGE_THRESHOLD: '82.9' steps: - name: Checkout Source From 4363e17d92ebbf1bdf55a8e839881cb5151d017b Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 11:09:40 -0600 Subject: [PATCH 10/11] Compare displayed (rounded) coverage to threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw coverage values from gcovr/coverage.py have more precision than the 1-decimal display (e.g. 82.875 → '82.9'). When a threshold was pinned to the displayed value, the raw comparison failed even though the displayed actual matched. Round the actual to 1 decimal before comparing so threshold checks line up with what's reported. Co-Authored-By: Claude Opus 4.7 --- dev/coverage.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dev/coverage.sh b/dev/coverage.sh index 23612a7b..31a96e20 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -183,21 +183,23 @@ cp -r _package/tests "$TESTS_COPY" PY_ACTUAL=$(python3 -c "import json; print(json.load(open('build/cov-py-summary.json'))['totals']['percent_covered'])") +# Round actuals to one decimal — both for display and for the threshold +# comparison so a threshold pinned to the displayed value (e.g. 82.9) does +# not fail because the unrounded actual is 82.875. +CPP_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$CPP_ACTUAL\"):.1f}')") +PY_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$PY_ACTUAL\"):.1f}')") + # Threshold checks. Both run unconditionally so the summary always reflects # real numbers; we set FAILED at the end if either layer is below. FAILED=0 -cpp_pass=$(python3 -c "import sys; sys.exit(0 if float('$CPP_ACTUAL') >= float('$CPP_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) -py_pass=$(python3 -c "import sys; sys.exit(0 if float('$PY_ACTUAL') >= float('$PY_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +cpp_pass=$(python3 -c "import sys; sys.exit(0 if float('$CPP_ACTUAL_DISP') >= float('$CPP_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +py_pass=$(python3 -c "import sys; sys.exit(0 if float('$PY_ACTUAL_DISP') >= float('$PY_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) [[ "$cpp_pass" == 0 ]] && FAILED=1 [[ "$py_pass" == 0 ]] && FAILED=1 cpp_status=$([[ "$cpp_pass" == 1 ]] && echo "pass" || echo "FAIL") py_status=$([[ "$py_pass" == 1 ]] && echo "pass" || echo "FAIL") -# Round actuals to one decimal for display. -CPP_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$CPP_ACTUAL\"):.1f}')") -PY_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$PY_ACTUAL\"):.1f}')") - echo echo "==> Threshold check" echo " C++: actual ${CPP_ACTUAL_DISP}% threshold ${CPP_COVERAGE_THRESHOLD}% [$cpp_status]" From c6c7c1bb55f6b593e34d8a0005cfa797af1d1af4 Mon Sep 17 00:00:00 2001 From: Gage Larsen Date: Fri, 8 May 2026 11:09:50 -0600 Subject: [PATCH 11/11] Lower both coverage thresholds to 70 Gives headroom over the current baseline (74.7% C++, 82.9% Python) so small fluctuations don't fail the job. Tighten later once the team agrees on a regression policy. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/Coverage.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml index 79ff7ed4..5a063914 100644 --- a/.github/workflows/Coverage.yaml +++ b/.github/workflows/Coverage.yaml @@ -40,8 +40,8 @@ jobs: CTEST_PARALLEL_LEVEL: '8' # Coverage thresholds. Pinned to current baseline; bump as coverage # improves. The job fails if either layer falls below its threshold. - CPP_COVERAGE_THRESHOLD: '74.7' - PY_COVERAGE_THRESHOLD: '82.9' + CPP_COVERAGE_THRESHOLD: '70' + PY_COVERAGE_THRESHOLD: '70' steps: - name: Checkout Source