diff --git a/.github/workflows/Coverage.yaml b/.github/workflows/Coverage.yaml new file mode 100644 index 00000000..5a063914 --- /dev/null +++ b/.github/workflows/Coverage.yaml @@ -0,0 +1,81 @@ +# Coverage workflow — hand-maintained, separate from xmsconan-generated CI. +# Lifts to xmsconan in Phase 2. +# +# Required repository secrets: +# CONAN2_USER_SECRET - Conan remote login username +# CONAN2_PASSWORD_SECRET - Conan remote login password +# AQUAPI_USERNAME_SECRET - devpi username for xmsconan install +# AQUAPI_PASSWORD_SECRET - devpi password for xmsconan install +# AQUAPI_URL_DEV - devpi index URL for xmsconan install + +name: Coverage + +on: + push: + pull_request: + +jobs: + coverage: + name: Coverage (Linux, GCC, Debug) + runs-on: ubuntu-latest + + container: + image: ghcr.io/aquaveo/conan-gcc13-py3.13:latest + + env: + LIBRARY_NAME: xmsgrid + XMS_VERSION: '0.0.0' + CONAN_REFERENCE: xmsgrid/0.0.0 + CONAN_ARCHS: x86_64 + CONAN_USERNAME: aquaveo + CONAN_CHANNEL: testing + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN2_USER_SECRET }} + CONAN_PASSWORD: ${{ secrets.CONAN2_PASSWORD_SECRET }} + CONAN_REMOTE_URL: https://conan2.aquaveo.com/artifactory/api/conan/aquaveo-stable + AQUAPI_USERNAME: ${{ secrets.AQUAPI_USERNAME_SECRET }} + AQUAPI_PASSWORD: ${{ secrets.AQUAPI_PASSWORD_SECRET }} + AQUAPI_URL: ${{ secrets.AQUAPI_URL_DEV }} + PYTHON_TARGET_VERSION: '3.13' + RELEASE_PYTHON: 'False' + CTEST_PARALLEL_LEVEL: '8' + # Coverage thresholds. Pinned to current baseline; bump as coverage + # improves. The job fails if either layer falls below its threshold. + CPP_COVERAGE_THRESHOLD: '70' + PY_COVERAGE_THRESHOLD: '70' + + steps: + - name: Checkout Source + uses: actions/checkout@v4 + + - name: Install Python Dependencies + run: | + pip install conan devpi-client wheel gcovr + pip install xmsconan>=2.12.2 -i https://public.aquapi.aquaveo.com/aquaveo/dev/+simple + + - name: Setup Conan + run: xmsconan_conan_setup --remote-url ${{ env.CONAN_REMOTE_URL }} --login + shell: bash + + - name: Run coverage + run: dev/coverage.sh + shell: bash + + - name: Upload HTML reports as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-html + path: | + build/coverage-html-cpp + build/coverage-html-py + retention-days: 14 + + - name: Upload Cobertura XML as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-xml + path: | + cov-cpp.xml + cov-py.xml + retention-days: 14 diff --git a/.gitignore b/.gitignore index 1bde17d2..911be830 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ conanfile.py build.py xms_conan2_file.py _package/pyproject.toml +.flake8 +pytest.ini # ── Build output ─────────────────────────────── build/ @@ -70,6 +72,11 @@ coverage_report/ *.profdata *.gcda *.gcno +.coverage-mark +.coverage-venv/ +.coverage +cov-cpp.xml +cov-py.xml # ── Local configuration ─────────────────────── .claude/ diff --git a/README.md b/README.md index 4d740554..8594fce6 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,23 @@ Documentation [C++ Documentation](https://aquaveo.github.io/xmsgrid/) [Python Documentation](https://aquaveo.github.io/xmsgrid/pydocs) + +Coverage +-------- + +Coverage runs on Linux/macOS via `dev/coverage.sh`. The script runs an +instrumented Debug build with the C++ test suite, then a release wheel build +with pytest in a clean venv. Outputs: + +- `cov-cpp.xml` / `cov-py.xml` — Cobertura XML. +- `build/coverage-html-cpp/` and `build/coverage-html-py/` — browsable HTML. + +Prerequisites: `gcovr` (`pip install gcovr`) and a working Conan toolchain +(see Building above). Phase 1 is GCC/Clang-only; Windows MSVC coverage is +not supported. + +CI runs the same script and uploads the HTML and XML reports as workflow +artifacts. Coverage is gated by simple percentage thresholds set in +`.github/workflows/Coverage.yaml` (`CPP_COVERAGE_THRESHOLD` and +`PY_COVERAGE_THRESHOLD`); the job fails if either layer falls below. +Initial values are `0` until a baseline is established. diff --git a/build.toml b/build.toml index c491f737..dd01d696 100644 --- a/build.toml +++ b/build.toml @@ -10,6 +10,18 @@ extra_cmake_text = """ if (NOT MSVC) add_compile_options(-ffp-contract=off) endif() + +# Coverage instrumentation, gated on the XMSGRID_COVERAGE environment variable. +# Set XMSGRID_COVERAGE=1 before invoking the build to add gcov flags to all +# targets. dev/coverage.sh is the canonical caller. +if (DEFINED ENV{XMSGRID_COVERAGE} AND NOT \"$ENV{XMSGRID_COVERAGE}\" STREQUAL \"\") + if (MSVC) + message(FATAL_ERROR \"XMSGRID_COVERAGE is GCC/Clang only\") + endif () + message(STATUS \"Coverage instrumentation enabled (XMSGRID_COVERAGE)\") + add_compile_options(--coverage -O0 -g) + add_link_options(--coverage) +endif () """ xms_dependencies = [{ name = "xmscore", version = "7.0.8" }] diff --git a/dev/coverage.sh b/dev/coverage.sh new file mode 100755 index 00000000..31a96e20 --- /dev/null +++ b/dev/coverage.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# Generate C++ and Python coverage reports for xmsgrid. +# +# C++ coverage comes from a Debug+testing Conan build instrumented with +# --coverage. ctest runs as part of the build; gcovr scans the resulting +# build folder in the conan cache. +# +# Python coverage comes from a separate pytest run against the built wheel +# in a clean venv with pytest-cov. Cross-credit (Python tests measuring C++ +# code) is not in scope for Phase 1. +# +# Outputs: +# cov-cpp.xml Cobertura, C++ +# cov-py.xml Cobertura, Python +# build/coverage-html-cpp/ Browsable C++ report +# build/coverage-html-py/ Browsable Python report +# +# Requirements: gcc/clang, gcovr, python, conan, xmsconan, gcc tools (gcov). + +set -euo pipefail + +REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +if [[ "$(uname -s)" == MINGW* || "$(uname -s)" == CYGWIN* ]]; then + echo "error: coverage is GCC/Clang-only; Windows is unsupported in Phase 1" >&2 + exit 1 +fi + +# Tool checks. +for tool in python gcovr gcov conan xmsconan_gen; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "error: required tool '$tool' not on PATH" >&2 + exit 1 + fi +done + +XMS_VERSION="${XMS_VERSION:-0.0.0}" + +echo "==> Generating xmsconan build artifacts" +xmsconan_gen --version "$XMS_VERSION" build.toml + +MARK="$REPO_ROOT/.coverage-mark" +trap 'rm -f "$MARK"' EXIT +touch "$MARK" +sleep 1 # Ensure subsequent file mtimes strictly exceed the marker. + +echo "==> C++ coverage build (Debug, testing=True, instrumented)" +XMSGRID_COVERAGE=1 python build.py \ + --filter='{"build_type":"Debug","testing":true}' + +echo "==> Python wheel build (Release, pybind=True, not instrumented)" +mkdir -p wheelhouse +python build.py \ + --filter='{"build_type":"Release","pybind":true}' \ + --wheel-dir wheelhouse + +echo "==> Locating Conan build folders that wrote .gcda files" +CONAN_HOME="${CONAN_HOME:-$HOME/.conan2}" +mapfile -t GCDA_FILES < <(find "$CONAN_HOME/p" -newer "$MARK" -name '*.gcda' 2>/dev/null || true) + +if [[ ${#GCDA_FILES[@]} -eq 0 ]]; then + echo "error: no .gcda files found under $CONAN_HOME/p; did the C++ build run?" >&2 + exit 1 +fi + +# Walk up from each .gcda to find the directory containing CMakeCache.txt +# (the build folder), then walk further up looking for the conan source +# folder — the first parent that contains a sibling 'xmsgrid' directory. +declare -A BUILD_TO_SOURCE=() +for gcda in "${GCDA_FILES[@]}"; do + dir="$(dirname "$gcda")" + while [[ "$dir" != "/" && ! -f "$dir/CMakeCache.txt" ]]; do + dir="$(dirname "$dir")" + done + if [[ ! -f "$dir/CMakeCache.txt" ]]; then + continue + fi + src_dir="$(dirname "$dir")" + while [[ "$src_dir" != "/" && ! -d "$src_dir/xmsgrid" ]]; do + src_dir="$(dirname "$src_dir")" + done + if [[ ! -d "$src_dir/xmsgrid" ]]; then + echo "error: could not locate a parent of $dir containing xmsgrid/" >&2 + exit 1 + fi + BUILD_TO_SOURCE["$dir"]="$src_dir" +done + +if [[ ${#BUILD_TO_SOURCE[@]} -eq 0 ]]; then + echo "error: could not locate any CMakeCache.txt above the discovered .gcda files" >&2 + exit 1 +fi + +echo "==> Found ${#BUILD_TO_SOURCE[@]} build folder(s):" +for bf in "${!BUILD_TO_SOURCE[@]}"; do + echo " build: $bf" + echo " source: ${BUILD_TO_SOURCE[$bf]}" +done + +mkdir -p build/coverage-html-cpp build/coverage-html-py + +# Thresholds. Default 0 means "do not fail". Override via env var to gate. +CPP_COVERAGE_THRESHOLD="${CPP_COVERAGE_THRESHOLD:-0}" +PY_COVERAGE_THRESHOLD="${PY_COVERAGE_THRESHOLD:-0}" + +GCOVR_COMMON=( + --filter '.*/xmsgrid/.*' + --exclude '.*\.t\.h$' + --exclude '.*/xmsgrid/python/.*' + --exclude '.*/_package/tests/.*' +) +GCOVR_OUTPUT=( + --xml-pretty --output cov-cpp.xml + --html-details "build/coverage-html-cpp/index.html" + --json-summary-pretty --json-summary "build/cov-cpp-summary.json" + --print-summary +) + +echo "==> Aggregating C++ coverage with gcovr (threshold: $CPP_COVERAGE_THRESHOLD%)" +# gcovr's --root must be the source folder; the build folder argument tells +# it where to find .gcda files. With one build folder we emit directly; +# with multiple we collect JSON intermediates then merge. Threshold check +# is performed below from the json-summary so the summary table can show +# the actual percentages even when a layer fails. +if [[ ${#BUILD_TO_SOURCE[@]} -eq 1 ]]; then + bf="${!BUILD_TO_SOURCE[*]}" + src="${BUILD_TO_SOURCE[$bf]}" + gcovr --root "$src" "${GCOVR_COMMON[@]}" "${GCOVR_OUTPUT[@]}" "$bf" +else + JSON_FILES=() + i=0 + for bf in "${!BUILD_TO_SOURCE[@]}"; do + src="${BUILD_TO_SOURCE[$bf]}" + out="build/coverage-cpp-$i.json" + gcovr --root "$src" "${GCOVR_COMMON[@]}" --json --output "$out" "$bf" + JSON_FILES+=(--add-tracefile "$out") + i=$((i + 1)) + done + # Merge: --root here only affects how merged paths are rendered, not + # which gcda files are read. Use the first source folder as the root. + first_src="${BUILD_TO_SOURCE[$(printf '%s\n' "${!BUILD_TO_SOURCE[@]}" | head -n1)]}" + gcovr --root "$first_src" "${GCOVR_COMMON[@]}" "${JSON_FILES[@]}" "${GCOVR_OUTPUT[@]}" +fi + +CPP_ACTUAL=$(python3 -c "import json; d=json.load(open('build/cov-cpp-summary.json')); print(d.get('line_percent', d.get('lines',{}).get('percent', 0)))") + +echo "==> Python coverage run (threshold: $PY_COVERAGE_THRESHOLD%)" +PY_VENV="$REPO_ROOT/.coverage-venv" +rm -rf "$PY_VENV" +python -m venv "$PY_VENV" +"$PY_VENV/bin/pip" install --quiet --upgrade pip +"$PY_VENV/bin/pip" install --quiet pytest pytest-cov +WHEEL=$(ls wheelhouse/*.whl | head -n1) +if [[ -z "$WHEEL" ]]; then + echo "error: no wheel found in wheelhouse/" >&2 + exit 1 +fi +# xmsgrid wheels declare an xmscore dependency that lives on Aquaveo devpi, +# not PyPI. Pass the public devpi index so pip can resolve it. +DEVPI_INDEX="${COVERAGE_PIP_INDEX:-https://public.aquapi.aquaveo.com/aquaveo/dev/+simple}" +"$PY_VENV/bin/pip" install --quiet --extra-index-url "$DEVPI_INDEX" "$WHEEL" + +# Copy tests outside the source tree before running pytest. Otherwise +# pytest's rootdir/sys.path discovery picks up the local _package/xms/grid/ +# source directory (which has no compiled _xmsgrid extension) and shadows +# the installed wheel, producing import errors. The conanfile's own python +# test runner does the same thing. +TESTS_COPY="$REPO_ROOT/build/coverage-tests" +rm -rf "$TESTS_COPY" +cp -r _package/tests "$TESTS_COPY" + +( + cd "$TESTS_COPY" + "$PY_VENV/bin/pytest" \ + --cov=xms.grid \ + --cov-report=xml:"$REPO_ROOT/cov-py.xml" \ + --cov-report=html:"$REPO_ROOT/build/coverage-html-py" \ + --cov-report=json:"$REPO_ROOT/build/cov-py-summary.json" \ + --cov-report=term \ + . +) + +PY_ACTUAL=$(python3 -c "import json; print(json.load(open('build/cov-py-summary.json'))['totals']['percent_covered'])") + +# Round actuals to one decimal — both for display and for the threshold +# comparison so a threshold pinned to the displayed value (e.g. 82.9) does +# not fail because the unrounded actual is 82.875. +CPP_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$CPP_ACTUAL\"):.1f}')") +PY_ACTUAL_DISP=$(python3 -c "print(f'{float(\"$PY_ACTUAL\"):.1f}')") + +# Threshold checks. Both run unconditionally so the summary always reflects +# real numbers; we set FAILED at the end if either layer is below. +FAILED=0 +cpp_pass=$(python3 -c "import sys; sys.exit(0 if float('$CPP_ACTUAL_DISP') >= float('$CPP_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +py_pass=$(python3 -c "import sys; sys.exit(0 if float('$PY_ACTUAL_DISP') >= float('$PY_COVERAGE_THRESHOLD') else 1)" && echo 1 || echo 0) +[[ "$cpp_pass" == 0 ]] && FAILED=1 +[[ "$py_pass" == 0 ]] && FAILED=1 + +cpp_status=$([[ "$cpp_pass" == 1 ]] && echo "pass" || echo "FAIL") +py_status=$([[ "$py_pass" == 1 ]] && echo "pass" || echo "FAIL") + +echo +echo "==> Threshold check" +echo " C++: actual ${CPP_ACTUAL_DISP}% threshold ${CPP_COVERAGE_THRESHOLD}% [$cpp_status]" +echo " Python: actual ${PY_ACTUAL_DISP}% threshold ${PY_COVERAGE_THRESHOLD}% [$py_status]" + +# Optional: append summary to GitHub Actions step summary. +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "## Coverage Summary" + echo + echo "| Layer | Threshold | Actual | Status |" + echo "|--------|-----------|--------|--------|" + echo "| C++ | ${CPP_COVERAGE_THRESHOLD}% | ${CPP_ACTUAL_DISP}% | $cpp_status |" + echo "| Python | ${PY_COVERAGE_THRESHOLD}% | ${PY_ACTUAL_DISP}% | $py_status |" + echo + echo "See \`gcovr\` and \`pytest --cov\` output above for actual percentages." + echo "Browseable HTML reports are uploaded as the \`coverage-html\` artifact." + } >> "$GITHUB_STEP_SUMMARY" +fi + +echo +echo "==> Coverage reports written:" +echo " cov-cpp.xml Cobertura, C++" +echo " cov-py.xml Cobertura, Python" +echo " build/coverage-html-cpp/ Browsable C++ report" +echo " build/coverage-html-py/ Browsable Python report" + +if [[ "$FAILED" -ne 0 ]]; then + echo + echo "error: coverage below threshold for one or more layers" >&2 + exit 1 +fi diff --git a/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md new file mode 100644 index 00000000..ec8b7031 --- /dev/null +++ b/docs/superpowers/specs/2026-05-08-coverage-reporting-design.md @@ -0,0 +1,297 @@ +# Coverage Reporting Design + +**Date:** 2026-05-08 +**Status:** Approved (design phase) +**Owner:** glarsen@aquaveo.com + +## Goal + +Add unified line-coverage reporting for both the C++ library and Python +bindings of `xmsgrid`. Reports are generated locally for developer iteration +and in CI for accountability. CI gates merges with simple percentage +thresholds rather than a third-party reporting service. Adopting Codecov or +similar is an open follow-up decision after a baseline is established. + +In Phase 1, the C++ and Python layers are measured independently: + +- **C++ coverage** is collected from a Debug+testing Conan build, instrumented + with `--coverage`, with ctest running the C++ test suite. +- **Python coverage** is collected from a separate pytest run against the + built wheel in a developer-controlled venv, using `pytest-cov`. + +Cross-credit (Python tests contributing to C++ coverage numbers) is **out of +scope for Phase 1** and is the headline feature of Phase 2 once the +orchestration is lifted into xmsconan. Three structural constraints in +xmsconan's generated `conanfile.py` make cross-credit costly to implement +without xmsconan changes: + +1. `BUILD_TESTING` and `IS_PYTHON_BUILD` are mutually exclusive in the + generated CMakeLists (`message(FATAL_ERROR ...)`). +2. `pybind=True` is forced to `build_type=Release` by `configure_options()`, + so there is no Debug pybind build available. +3. The pytest run inside `conanfile.build()` creates a venv with only + `numpy, wheel, pytest` — `pytest-cov` is not installed. + +Phase 1 sidesteps all three by treating the suites as independent. Phase 2 +addresses them in xmsconan. + +## Non-Goals + +- Windows / MSVC coverage. gcov is GCC/Clang only; instrumenting MSVC + requires OpenCppCoverage or clang-cl source-based coverage and is a + separate effort. +- Branch coverage. Line coverage only for now. +- Hard coverage gates that block PRs. Reporting is informational; tightening + is a follow-up. +- Refactoring xmsconan. Phase 1 is repo-local. Generic pieces lift in Phase 2. + +## Architecture + +``` +┌─ C++ Coverage Build (Linux, GCC, Debug, testing=True) ───────┐ +│ XMSGRID_COVERAGE=1 → CMake adds --coverage to all targets │ +│ python build.py --filter='{"build_type":"Debug", │ +│ "testing":true}' │ +│ conanfile.run_cxx_tests() → ctest → *.gcda in conan cache │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ gcovr scans conan build │ + │ folder → cobertura-cpp.xml │ + └─────────────────────────────┘ + +┌─ Python Coverage Run (Linux, against built wheel) ───────────┐ +│ python build.py --filter='{"build_type":"Release", │ +│ "pybind":true}' │ +│ --wheel-dir wheelhouse │ +│ Then in a clean venv: │ +│ pip install wheelhouse/*.whl pytest pytest-cov │ +│ pytest --cov=xms.grid --cov-report=xml _package/tests │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ coverage.py → cov-py.xml │ + └─────────────────────────────┘ + + │ + ▼ + ┌─────────────────────────────┐ + │ Threshold check │ + │ gcovr --fail-under-line │ + │ pytest --cov-fail-under │ + └─────────────────────────────┘ +``` + +The C++ build is instrumented; the Python wheel build is **not**. Python +coverage measures Python-level wrappers in `_package/xms/grid/` only. +Cross-instrumentation is Phase 2. + +The reports (Cobertura XML + browsable HTML) upload as workflow artifacts +on every CI run regardless of pass/fail, so reviewers can always inspect +the numbers. If the team later decides to adopt Codecov or a similar +service, the cobertura XMLs are already in the right format to drop in. + +## Components + +### 1. CMake option (in `build.toml` `extra_cmake_text`) + +```cmake +if(DEFINED ENV{XMSGRID_COVERAGE} AND NOT "$ENV{XMSGRID_COVERAGE}" STREQUAL "") + if(MSVC) + message(FATAL_ERROR "Coverage build is GCC/Clang only") + endif() + message(STATUS "Coverage instrumentation enabled (XMSGRID_COVERAGE)") + add_compile_options(--coverage -O0 -g) + add_link_options(--coverage) +endif() +``` + +Gated on an environment variable instead of a CMake cache variable so the +coverage script can flip it without needing to pass `--cmake-args` through +xmsconan's `build.py`. No xmsconan generator change required. The variable +flows naturally through Conan's `tools.env.virtualenv` and into the CMake +configure step. + +### 2. `dev/coverage.sh` + +POSIX shell script (Linux + macOS). Single entry point. Steps: + +1. Drop a marker file (`.coverage-mark`) — used to identify build folders + that this run wrote `.gcda` files into. +2. Run the C++ coverage build: + ``` + XMSGRID_COVERAGE=1 python build.py \ + --filter='{"build_type":"Debug","testing":true}' + ``` + The conanfile's `run_cxx_tests()` runs ctest as part of `build()`, so + this single invocation builds, runs, and emits `.gcda` files into + `~/.conan2/p//b/build/...`. +3. Run the wheel build: + ``` + python build.py \ + --filter='{"build_type":"Release","pybind":true}' \ + --wheel-dir wheelhouse + ``` + No coverage env var here — the wheel is for Python coverage only and + does not need instrumentation in Phase 1. +4. Discover C++ build folders that received fresh `.gcda` files: + ``` + find ~/.conan2/p -newer .coverage-mark -name '*.gcda' + ``` + Walk up from each match to the directory containing `CMakeCache.txt`; + that's the build folder gcovr needs to scan. Deduplicate the set. +5. Run gcovr against the source root with each discovered build folder: + ``` + gcovr --root . \ + --filter 'xmsgrid/' \ + --exclude '.*\.t\.h$' \ + --exclude 'xmsgrid/python/.*' \ + --exclude '_package/tests/.*' \ + --xml cov-cpp.xml \ + --html-details build/coverage-html-cpp/index.html \ + + ``` + When multiple build folders are found (rare in Phase 1, but possible + if shards are used), emit a JSON intermediate per folder via + `--json-summary`, then merge with `gcovr --add-tracefile a.json -a b.json`. +6. Run Python tests with coverage in a clean venv, gated by the Python + threshold: + ``` + python -m venv .coverage-venv + .coverage-venv/bin/pip install wheelhouse/*.whl pytest pytest-cov + .coverage-venv/bin/pytest \ + --cov=xms.grid \ + --cov-report=xml:cov-py.xml \ + --cov-report=html:build/coverage-html-py \ + --cov-fail-under="$PY_COVERAGE_THRESHOLD" \ + _package/tests + ``` +7. The C++ gcovr invocation in step 5 passes + `--fail-under-line "$CPP_COVERAGE_THRESHOLD"` so it exits non-zero when + the C++ percentage falls below. +8. When running under GitHub Actions (`$GITHUB_STEP_SUMMARY` set), append + a short markdown summary noting the configured thresholds. The actual + percentages appear in the gcovr / pytest output above the summary. + +Thresholds default to `0` (no gating) via env vars +`CPP_COVERAGE_THRESHOLD` and `PY_COVERAGE_THRESHOLD`. CI sets them +explicitly in `Coverage.yaml`; locally they can be overridden via env. + +A `dev/coverage.bat` is **not** added in Phase 1; Windows coverage is out +of scope. + +### 3. `.github/workflows/Coverage.yaml` + +Hand-maintained, separate from the xmsconan-generated `XmsGrid-CI.yaml`. +Single job, Linux, GCC, Debug: + +- Triggers: `push`, `pull_request`. +- Checks out source, installs xmsconan + gcovr + Python deps, sets up Conan + login. +- Sets `CPP_COVERAGE_THRESHOLD` and `PY_COVERAGE_THRESHOLD` (initially `0`, + bumped after the first run establishes a baseline). +- Runs `dev/coverage.sh`. The script's gcovr / pytest steps fail the job + when either threshold is missed. +- Uploads `cov-cpp.xml` and `cov-py.xml` as the `coverage-xml` artifact and + the HTML reports as the `coverage-html` artifact, with `if: always()` so + they are available even on threshold failure. + +The job uses the same Conan / devpi secrets as the existing CI workflow. +No reporting-service token is required. + +### 4. README update + +Short `## Coverage` section: how to run `dev/coverage.sh`, where the HTML +lands, how thresholds are configured. + +## Repo Setup (one-time, manual) + +None for Phase 1 beyond the secrets the existing workflow already requires. +The first CI run establishes a baseline; bump the threshold env values in +`Coverage.yaml` afterwards to enforce no-regression. + +## Reporting Service Decision (deferred) + +Whether to adopt Codecov or a similar service — for diff coverage, +trend tracking, or PR comments — is intentionally deferred to a follow-up +decision after the team sees a few weeks of threshold-gated runs and can +judge whether richer signal is worth the integration. The cobertura XMLs +emitted by Phase 1 are already in the standard format any of these +services accept, so adopting one later is mechanical. + +## Phase 2 — Lift to xmsconan + +After Phase 1 ships and stabilizes (a few real PRs through the new +workflow, thresholds set above zero, no flaky failures for a week), lift +the generic pieces into xmsconan in a follow-up PR. + +**Lifted into xmsconan:** + +- The CMake coverage option block — emitted by `xmsconan_gen` into the + generated CMakeLists, gated by a flag in `build.toml`. +- `Coverage.yaml` workflow — generated by `xmsconan_gen` alongside the + existing `-CI.yaml`, gated by the same flag. +- Orchestration shipped as a console entry point: `xmsconan_coverage`, + alongside `xmsconan_gen`, `xmsconan_wheel_repair`, `xmsconan_wheel_deploy`. + The repo-local `dev/coverage.sh` shrinks to a one-liner or is deleted in + favor of documenting `xmsconan_coverage`. +- Default gcovr exclusion patterns: `*.t.h`, `_package/tests/`, + `/python/`. + +**Stays repo-local:** + +- A toggle in `build.toml`: + ```toml + [ci] + enable_coverage = true + ``` +- `codecov.yml` (per-repo thresholds and ignore lists). +- `CODECOV_TOKEN` repo secret. + +**Migration of xmsgrid in Phase 2:** + +1. Bump xmsconan dependency in `build.toml` to the version with coverage + support. +2. Set `enable_coverage = true` in `build.toml`. +3. Replace `dev/coverage.sh` body with a call to `xmsconan_coverage`, or + delete the script and document `xmsconan_coverage` directly. +4. Delete `.github/workflows/Coverage.yaml` — now regenerated. +5. Re-run `xmsconan_gen`. +6. Verify the next CI run produces the same Codecov report. + +**Why two phases:** + +- Phase 1 lets us iterate on script details, gcovr filters, and Codecov + config without an xmsconan release on every change. +- Phase 2 is mechanical: copy proven files into xmsconan templates, add + the `build.toml` knob. + +A feature issue tracking Phase 2 will be filed in the xmsconan repo when +Phase 1 lands. + +## Testing + +- Unit testing the coverage script itself is overkill — it's an + orchestrator. Validation is end-to-end: a successful CI run on a PR + shows non-zero numbers for both flags in the Codecov comment. +- Local validation: run `dev/coverage.sh` on a clean checkout, confirm + HTML reports open and show non-zero coverage for at least one file in + each layer. + +## Risks & Open Questions + +- **Pybind11 coverage instrumentation overhead.** `--coverage` adds + significant build time and binary size. Linux Debug coverage build is + expected to take notably longer than the existing Linux Debug CI job; + this is acceptable since coverage is its own job and runs in parallel. +- **gcda path mismatch.** If pytest is run from a different working + directory than the one where the extension was built, gcov may not + find source files. The script pins `cwd` to the repo root throughout. +- **Codecov free-tier rate limits.** Public repos are unlimited on + uploads; no concern for xmsgrid specifically. +- **Linux-only signal.** Coverage numbers reflect what runs on Linux. For + a math/geometry library this is fine — platform-specific code is + minimal — but anything truly Windows- or Mac-only will show as + uncovered. Acceptable.