diff --git a/README.md b/README.md index 2921030f..c6c229d4 100644 --- a/README.md +++ b/README.md @@ -23,32 +23,17 @@ eval "$(poetry env activate)" ### Optional extras -The base install is intentionally lean — heavy plotting and ML dependencies are -opt-in via Poetry extras. Install only what a given workflow needs: - -| Extra | Enables | Pulls in | -| --- | --- | --- | -| `viz` | Static and interactive plotting | matplotlib, seaborn, plotly | -| `interactive` | Interactive plots / dashboards | plotly | -| `duckdb` | DuckDB-backed queries | duckdb | -| `hgc` | Joint genotyping plots (genotype adjustment needs no extra) | matplotlib, seaborn | -| `psroc` | Pathogenicity Score ROC analysis | matplotlib, plotly, scikit-learn, scipy | -| `ptm` | CPTAC proteomics builders | cptac, sorted-nearest | -| `constraint` | Tissue-specificity / constraint metrics | tspex, matplotlib, seaborn, scipy | -| `enrichex` | `hvantk enrichex overlap` / `burden` and their plots | scipy, matplotlib, seaborn | -| `cohort` | `hvantk cohort burden` (Fisher gene burden) | scipy | -| `ancestry` | Ancestry inference (PCA + Random Forest + plots) | scikit-learn, matplotlib, seaborn, scipy | -| `ml` | scikit-learn-backed features only | scikit-learn, scipy | -| `expression` | `hvantk expression summarize` / `markers`, and `ptm constraint --expression-metric mean` | scanpy, scipy | +The base install is intentionally lean — plotting, machine-learning, and a few +provider-specific dependencies are opt-in Poetry extras: ```bash -# One or more extras at once -poetry install --extras "ancestry psroc" - -# Or a single extra -poetry install --extras ml +poetry install --extras "ancestry psroc" # one or more +poetry install --all-extras # everything ``` +For the full table — what each extra pulls in and which commands need it — see +[Installation → Optional features](docs_site/getting-started/installation.md#optional-features-extras). + Verify it works: ```bash @@ -122,105 +107,24 @@ returns `(native_obj, Provenance)` zero-cost. ### Plugin contract — adding a new data source -Each plugin under `hvantk/skills//` declares itself via -[`plugin.yaml`](hvantk/skills/clinvar/plugin.yaml) and provides a builder -that returns a typed artifact: - -```python -# hvantk/skills/clinvar/builder.py -def build_clinvar(parsed_input, ctx: BuildContext, **params) -> AnnotationTable: - ht = hl.import_vcf(str(parsed_input), force=True, ...).rows().key_by("locus", "alleles") - return AnnotationTable.from_hail( - ht, provenance=ctx.provenance(schema_id="clinvar-variants-v1") - ) -``` - -The platform orchestrator [`run_builder_for_spec`](hvantk/core/plugin/run_builder.py) -ties it all together at build time: - -```mermaid -sequenceDiagram - participant CLI as hvantk reprocess - participant Reg as plugin registry - participant Probe as drift_probe() - participant Build as build_fn(parsed, ctx) - participant IO as core/io - - CLI->>Reg: get_dataset("clinvar:variants") - Reg-->>CLI: DatasetSpec (lazy bind on first access) - CLI->>Probe: compute source fingerprint - Probe-->>CLI: probe dict - CLI->>CLI: BuildContext(plugin, version, fingerprint, …) - CLI->>Build: (parsed_input, ctx, **params) - Build-->>CLI: AnnotationTable(provenance=ctx.provenance(schema_id=…)) - CLI->>CLI: validate artifact_type + schema_id - CLI->>IO: artifact.save(path) - IO-->>IO: write data + sidecar .provenance.json -``` - -Twenty-one plugins ship today: `clinvar`, `clingen`, `gencc`, `gwas-catalog`, -`hgnc`, `gtex-eqtl`, `insider`, `msigdb`, `uniprot-ptm`, `peptideatlas`, -`expression-atlas`, `cptac`, `ucsc-cellbrowser`, `gevir`, `gnomad-metrics`, -`ensembl-gene`, `dbnsfp`, `cosmic-cgc`, `pqtl`, `alphagenome`, `onek-genomes`. - -### Project structure - -``` -hvantk/ -├── core/ # platform substrate — stable contracts -│ ├── models/ # AnnotationTable, ExpressionMatrix, VariantMatrix, GeneSet, -│ │ # Provenance, BuildContext, Expr DSL, -│ │ # AlgorithmMeta (@algorithm decorator) -│ ├── io/ # save / load / save_native / load_native, -│ │ # sidecar provenance manifests, legacy shim -│ ├── plugin/ # plugin registry, run_builder_for_spec, -│ │ # two-pass discovery (DatasetManifest → DatasetSpec) -│ ├── tool/ # tool manifest discovery (descriptive) -│ ├── streamers/ # Streamer ABCs — query/iterate built tables -│ │ # (concrete subclasses live in skills//) -│ ├── ontology/ # OBO / MONDO parsers -│ └── utils/ # generic helpers (hail context, hail_helpers, -│ # file utils, gene sets) -│ -├── algorithms/ # analytics — consume artifacts, return artifacts -│ ├── ancestry/ # PCA + Random Forest ancestry inference -│ ├── enrichex/ # gene set enrichment + burden testing -│ ├── expression/ # tissue specificity (tau, gini, etc.) -│ ├── hgc/ # joint genotyping (gvcf combine, VDS, QC) -│ ├── ptm/ # PTM coordinate mapping + atlas -│ ├── psroc/ # pathogenicity score ROC analysis -│ ├── qtlcascade/ # eQTL → pQTL cascade + colocalization -│ ├── annotation/ # spine / prepare / compose annotation pipeline -│ ├── burden/, cohort/ # rare-variant burden + external cohort handling -│ ├── rerank/ # multi-omic gene re-ranking (feature axes + audit) -│ ├── statistics/ # multiple-testing correction, shared stats -│ └── visualization/ # shared figure helpers (empty_figure, save_figure) -│ -├── skills/ # data-source plugins (21 total) -│ ├── / -│ │ ├── plugin.yaml # declarative manifest (drives discovery + CLI) -│ │ ├── builder.py # Phase B: (parsed, ctx) → AnnotationTable / … -│ │ ├── drift_probe.py # upstream fingerprint -│ │ ├── cli.py # downloader (auto-wired via manifest cli: block) -│ │ └── tests/ # per-plugin conformance tests + fixtures -│ └── _conventions/SKILL.md # contract documentation -│ -├── tools/ # CLI wiring + workflow orchestration -│ ├── plugins/ # download, drift, reprocess, plugins/tools list -│ ├── hgc/ # joint-genotyping CLI (lazy-loaded) -│ ├── infra/ # catalog, utils (check-install, bgzf) -│ ├── annotation/ # annotate spine / prepare / compose -│ ├── cohort/ # cohort validate / burden / attach -│ ├── rerank/ # rerank CLI -│ ├── genesets/ # gene set extraction / preparation -│ ├── training_sets/ # TrainingSetBuilder — library only, no CLI command -│ └── ancestry/, enrichex/, expression/, ptm/, qtl/ # one package per domain -│ -├── resources/ # platform metadata (unified catalog registry) -└── tests/ # cross-cutting tests (dependency directions, - # plugin conformance, io round-trips, - # Expr algebra parity, etc.) -``` +Each data source ships as a self-contained plugin under `hvantk/skills//`, +declared by a [`plugin.yaml`](hvantk/skills/clinvar/plugin.yaml) manifest naming its +builder and drift probe, plus an optional downloader for sources that permit an +automated fetch. Sources behind a license gate, or too large to mirror, ship a +documented acquisition procedure instead. The platform orchestrator +[`run_builder_for_spec`](hvantk/core/plugin/run_builder.py) resolves the manifest, +computes the source fingerprint, calls the builder, validates the returned artifact +against the manifest's `artifact_type` and `schema_id`, and saves it alongside a +sidecar `.provenance.json`. The loader discovers manifests on its own — there is no +registry to edit. + +The full contract and the annotated directory tree live in the architecture guide: + +- [Plugin contract](docs_site/architecture.md#3-plugin-contract--adding-a-data-source) + — build sequence diagram, annotated `plugin.yaml`, two-pass loader, streamer + placement rule +- [Project structure](docs_site/architecture.md#project-structure) — what lives in + each package, layer by layer ### How to extend diff --git a/containers/hvantk.def b/containers/hvantk.def new file mode 100644 index 00000000..46fad9b7 --- /dev/null +++ b/containers/hvantk.def @@ -0,0 +1,58 @@ +Bootstrap: docker +From: python:3.10-slim-bullseye + +# hvantk container image, built from the committed poetry.lock +# (hail 0.2.137 + pyspark 3.5.8). See docs_site/guide/hpc-migration.md section 3. +# +# BASE IMAGE: bullseye, not bookworm. Debian 12 (bookworm) has no openjdk-11 package +# at all -- apt reports it is replaced by openjdk-17-jre-headless, and Hail 0.2.x / +# Spark 3.5 support Java 8 or 11 ONLY. Debian 11 (bullseye) still ships +# openjdk-11-jdk-headless. Do not "modernise" this base without checking `java -version` +# inside the built image. +# +# EXTRAS: checked against [project.optional-dependencies] in pyproject.toml. This set +# covers every unique package across all extras. `expression` is required or +# `hvantk expression ...` has no scanpy. `ml` and `interactive` are deliberately +# omitted as redundant: scikit-learn arrives via ancestry/psroc, plotly via viz. + +%files + pyproject.toml /opt/hvantk/pyproject.toml + poetry.lock /opt/hvantk/poetry.lock + hvantk /opt/hvantk/hvantk + README.md /opt/hvantk/README.md + +%post + set -e + # --- Java 11 (Hail 0.2.137 / Spark 3.5 support Java 8 or 11 ONLY) + native libs --- + apt-get update && apt-get install -y --no-install-recommends \ + openjdk-11-jdk-headless \ + build-essential g++ \ + zlib1g-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libdeflate-dev \ + libopenblas-dev liblapack-dev liblz4-dev libhdf5-dev git + # --- hvantk from the committed lock (reproducible; not unpinned pip install) --- + pip install --no-cache-dir "poetry>=2.0" + cd /opt/hvantk + poetry config virtualenvs.create false + poetry install --no-interaction --no-root \ + --extras "hgc ptm ancestry psroc constraint enrichex cohort viz duckdb expression" + poetry install --no-interaction --only-root + apt-get purge -y build-essential g++ git && apt-get autoremove -y + apt-get clean && rm -rf /var/lib/apt/lists/* + # fail the build here rather than shipping a broken image + java -version 2>&1 | head -1 + python -c "import hail; print('hail', hail.__version__)" + +%environment + export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + export LC_ALL=C.UTF-8 LANG=C.UTF-8 + export NO_PROXY=localhost,127.0.0.1,0.0.0.0,::1 + export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 + +%runscript + exec hvantk "$@" + +%labels + org hvantk + hail 0.2.137 + pyspark 3.5.8 diff --git a/containers/hvantk_run.sh b/containers/hvantk_run.sh new file mode 100755 index 00000000..e99f072d --- /dev/null +++ b/containers/hvantk_run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Run hvantk from the Apptainer/Singularity image with Spark scratch set up correctly. +# +# Without SPARK_LOCAL_DIRS on a writable NODE-LOCAL dir, Hail init inside the container +# fails with "DiskBlockManager: Failed to create any local dir" followed by a misleading +# "[Errno 111] Connection refused" from py4j. Always go through this wrapper. +# +# bash hvantk_run.sh utils check-install +# bash hvantk_run.sh reprocess clinvar:variants --raw-dir ... --output ... +# HVANTK_SIF=/path/to/hvantk.sif bash hvantk_run.sh --help +set -euo pipefail + +HVANTK_SIF="${HVANTK_SIF:-${WORK:?WORK unset}/containers/hvantk.sif}" +[[ -f "$HVANTK_SIF" ]] || { echo "ERROR: image not found: $HVANTK_SIF" >&2; exit 1; } + +# node-local scratch, never home/Lustre/GPFS (guide section 4.1) +SPARK_SCRATCH="${SLURM_TMPDIR:-/tmp/spark-$USER-${SLURM_JOB_ID:-$$}}" +mkdir -p "$SPARK_SCRATCH" +cleanup(){ rm -rf "$SPARK_SCRATCH"; } +trap cleanup EXIT + +# Bind any extra data roots the command needs, e.g. HVANTK_BIND="$WORK:$WORK" +BINDS=(-B "$SPARK_SCRATCH:$SPARK_SCRATCH") +[[ -n "${HVANTK_BIND:-}" ]] && BINDS+=(-B "$HVANTK_BIND") + +exec singularity exec "${BINDS[@]}" \ + --env SPARK_LOCAL_DIRS="$SPARK_SCRATCH" \ + --env TMPDIR="$SPARK_SCRATCH" \ + --env HAIL_TMPDIR="$SPARK_SCRATCH" \ + "$HVANTK_SIF" hvantk "$@" diff --git a/docs_site/architecture.md b/docs_site/architecture.md index b659c6f1..7177eed3 100644 --- a/docs_site/architecture.md +++ b/docs_site/architecture.md @@ -294,6 +294,13 @@ def build_my_source_variants(parsed_input, ctx: BuildContext, **params) -> Annot ) ``` +A shipped one to copy from: `hvantk/skills/clinvar/builder.py` defines +`build_clinvar`, which `hvantk/skills/clinvar/plugin.yaml` binds to the +`clinvar:variants` dataset under schema `clinvar-variants-v1`. The builder name +is whatever `plugin.yaml`'s `builder.function` declares — plugins in the tree use +both the short form (`build_clinvar`) and the per-dataset form +(`build_clingen_gene_disease`). + The plugin loader (`hvantk/core/plugin/loader.py`) discovers manifests via a **two-pass mechanism**: diff --git a/docs_site/guide/data-sources.md b/docs_site/guide/data-sources.md index 500c6a1e..ef05c5ba 100644 --- a/docs_site/guide/data-sources.md +++ b/docs_site/guide/data-sources.md @@ -179,12 +179,22 @@ hvantk reprocess gnomad-metrics:metrics --skip-download \ --plugin-arg key=transcript ``` -### INSIDER interactome (~100 MB) +### INSIDER interactome (~1.2 GB genomic BED; ~49 MB pair table) -Protein-protein interaction sites from the INSIDER database. +Protein-protein interaction interface residues from the INSIDER database. URL: http://interactomeinsider.yulab.org/downloads.html -**Download**: Visit http://interactomeinsider.yulab.org/downloads.html and download the interaction site BED file. +INSIDER ships **two** products, and hvantk builds a dataset from each: + +| dataset | file | size | direct URL | +| --- | --- | --- | --- | +| `insider:variants` | `Whole_Human_Interactome_Interface_hg38.bed` | ~1.17 GB | `http://interactomeinsider.yulab.org/bed/all.bed` | +| `insider:interfaces` | `H_sapiens_interfacesALL.txt` | ~49 MB | `http://interactomeinsider.yulab.org/downloads/interfacesALL/H_sapiens_interfacesALL.txt` | + +**Download**: the downloads page carries no links in its markup, so use the direct +URLs above (they are also recorded in the plugin catalog — `hvantk catalog show +INSIDER_v1.0`). Both are served over plain HTTP; the site has no HTTPS listener. +The BED is >1 GB, so acquisition is manual per the downloader framework in CLAUDE.md. **Build**: @@ -239,11 +249,25 @@ pathogenicity score. Abramovs, Brass & Tassabehji, 2020, Nature Genetics 52(1):35-39 (PMID 31873297, DOI 10.1038/s41588-019-0560-2). URL: https://www.nature.com/articles/s41588-019-0560-2 -**Download**: Small supplementary table from the Nature Genetics publication -(https://www.nature.com/articles/s41588-019-0560-2) or the authors' repository -(https://github.com/gevirank/gevir). At ~1-2 MB with a stable, public URL, GeVIR -qualifies for a real downloader under the framework in CLAUDE.md — a recommended -follow-up (not yet implemented). +**Download**: the metric table is **Supplementary Table 2** of the Nature Genetics +paper, served as the article's MOESM3 object: + +``` +https://static-content.springer.com/esm/art%3A10.1038%2Fs41588-019-0560-2/MediaObjects/41588_2019_560_MOESM3_ESM.xlsx +``` + +That is a ~10.3 MB `.xlsx` workbook (only MOESM3 of the six supplementary slots is +public; the rest return 403). The builder reads a bgzipped TSV, so extract sheet +`table_2` and BGZF-compress it before building. + +> **Note:** the authors' repository at https://github.com/gevirank/gevir ships the +> **analysis code only** — its `tables/` directory holds a placeholder file — so it +> is not a source for the metric table. Earlier revisions of this guide pointed +> there. + +A real downloader would have to do the extract-and-convert step, not just fetch the +URL, so it is more than the usual thin wrapper — a recommended follow-up (not yet +implemented). **Build**: diff --git a/docs_site/guide/hpc-migration.md b/docs_site/guide/hpc-migration.md index 2f42b677..a38932d7 100644 --- a/docs_site/guide/hpc-migration.md +++ b/docs_site/guide/hpc-migration.md @@ -125,90 +125,77 @@ the lock; do not regenerate or loosen it. And: ### 3.2 `hvantk.def` (build from the locked environment) -Place this at the repo root (or under `containers/`). It installs the **exact** -locked dependency set, not unpinned `pip install hail`. - -```singularity -Bootstrap: docker -From: python:3.10-slim-bookworm - -%files - pyproject.toml /opt/hvantk/pyproject.toml - poetry.lock /opt/hvantk/poetry.lock - hvantk /opt/hvantk/hvantk - README.md /opt/hvantk/README.md - -%post - set -e - # --- Java 11 (Hail 0.2.137 requirement; NOT 17+) + native build/runtime libs --- - apt-get update && apt-get install -y --no-install-recommends \ - openjdk-11-jdk-headless \ - build-essential g++ \ - zlib1g-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libdeflate-dev \ - libopenblas-dev liblapack-dev liblz4-dev libhdf5-dev git - # --- hvantk via Poetry, from the committed lock (reproducible) --- - # >=2.0: pyproject.toml uses PEP 621 [project] metadata, which poetry 1.8 cannot read. - pip install --no-cache-dir "poetry>=2.0" - cd /opt/hvantk - poetry config virtualenvs.create false - # install the locked deps + the extras you actually run (trim as needed): - poetry install --no-interaction --no-root \ - --extras "hgc ptm ancestry psroc constraint enrichex cohort viz duckdb" - poetry install --no-interaction --only-root - # experiment-only extras NOT in pyproject (e.g. PTM functionality pilot): - pip install --no-cache-dir pyBigWig - apt-get purge -y build-essential g++ git && apt-get autoremove -y - apt-get clean && rm -rf /var/lib/apt/lists/* - -%environment - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 - export PATH=$JAVA_HOME/bin:$PATH - export LC_ALL=C.UTF-8 LANG=C.UTF-8 - # keep Hail/py4j localhost traffic off any proxy - export NO_PROXY=localhost,127.0.0.1,0.0.0.0,::1 - # avoid BLAS thread explosion under Spark - export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 - -%runscript - exec hvantk "$@" - -%labels - org hvantk - hail 0.2.137 -``` - -> Trim `--extras` to what you run. The core install omits scikit-learn / scipy / viz / -> cptac / tspex / duckdb / scanpy — they live behind extras (`hgc`, `ptm`, -> `ancestry`, `psroc`, `constraint`, `enrichex`, `cohort`, `ml`, `viz`, `duckdb`, -> `interactive`, `expression`). The authoritative list is -> `[project.optional-dependencies]` in `pyproject.toml`; this one is prose and is not -> machine-checked. -> `pyBigWig` is **not** a hvantk dependency — include it only for experiments that -> query bigWig tracks. Genotype adjustment needs **no** extra: `annotate_adj` is -> ported in-tree, so `gnomad` is no longer a dependency at all. +The definition file lives in the repo at **`containers/hvantk.def`** +— build from that file rather than copying a snippet, so the base image and extras +cannot drift from what was last built and validated. It installs the **exact** locked +dependency set, not an unpinned `pip install hail`. + +Two choices in it are load-bearing and must not be "modernised" without re-validating: + +- **Base image is `python:3.10-slim-bullseye`, not bookworm.** Debian 12 (bookworm) has + no `openjdk-11` package at all — `apt-get install openjdk-11-jdk-headless` fails with + *"Package 'openjdk-11-jdk-headless' has no installation candidate … the following + packages replace it: openjdk-17-jre-headless"*. Hail 0.2.x / Spark 3.5 support Java 8 + or 11 **only**, so a bookworm base either fails the build or silently gives you + Java 17. Debian 11 (bullseye) ships `openjdk-11-jdk-headless` (11.0.32.1). +- **Extras include `expression`.** The set + `hgc ptm ancestry psroc constraint enrichex cohort viz duckdb expression` + is checked against `[project.optional-dependencies]` in `pyproject.toml` and covers + every unique package across all extras. Omitting `expression` builds an image with no + scanpy, so `hvantk expression …` cannot run. `ml` and `interactive` are redundant: + scikit-learn arrives via `ancestry`/`psroc`, plotly via `viz`. + +The `%post` block ends by printing `java -version` and importing `hail`, so a broken +image fails at **build** time rather than at first use. ### 3.3 Build (rootless) and validate ```bash # Keep build cache/tmp OFF the small home quota -export APPTAINER_CACHEDIR=/scratch/$USER/apptainer/cache -export APPTAINER_TMPDIR=/scratch/$USER/apptainer/tmp -mkdir -p "$APPTAINER_CACHEDIR" "$APPTAINER_TMPDIR" +export SINGULARITY_CACHEDIR=$WORK/containers/cache # APPTAINER_* if you have apptainer +export SINGULARITY_TMPDIR=$WORK/containers/tmp +mkdir -p "$SINGULARITY_CACHEDIR" "$SINGULARITY_TMPDIR" -# Build on the cluster if --fakeroot is allowed: -apptainer build --fakeroot hvantk.sif hvantk.def -# ...or build on a laptop / CI and copy the .sif over via Globus/rsync. +# Build from the REPO ROOT: %files paths in the def are relative to the build CWD. +cd +singularity build $WORK/containers/hvantk.sif containers/hvantk.def +``` -# Validate the full native + JVM stack on a COMPUTE node (not login): -srun -c 4 --mem 16g apptainer exec hvantk.sif hvantk utils check-install +**`--fakeroot` is not usable on every cluster.** It needs a `/etc/subuid` entry for your +account; without one the build fails immediately with +`could not use fakeroot: no valid mapping entry found for `, and a plain +unprivileged build is refused outright with +`--remote, --fakeroot, or the proot command are required to build this source as a +non-root user`. Check before assuming: + +```bash +grep "^$USER:" /etc/subuid # empty output => --fakeroot will NOT work here +``` + +When there is no `subuid` entry, SingularityCE 3.11+/4.x accepts a static **`proot`** +instead, which needs no privileges at all: + +```bash +mkdir -p ~/bin && curl -fsSL -o ~/bin/proot https://proot.gitlab.io/proot/bin/proot +chmod +x ~/bin/proot +export PATH="$HOME/bin:$PATH" # singularity picks proot up from PATH +singularity build $WORK/containers/hvantk.sif containers/hvantk.def +``` + +Builds take roughly 10 minutes and the image is ~2.2 GB. + +Validate the full native + JVM stack on a **compute** node (not login) — and go through +the run wrapper, because Hail cannot initialise inside the container without Spark +scratch (see §4.1): + +```bash +srun -c 4 --mem 16g bash containers/hvantk_run.sh utils check-install # expects: Hail version prints, balding_nichols_model smoke test passes. ``` `hvantk utils check-install` is the canonical go/no-go for a node — it initializes Hail, prints `hl.version()`, runs a Hail smoke test, and diagnoses proxy problems. ---- - ## 4. Running Hail on the cluster ### 4.1 Default: Spark local mode on one exclusive fat node @@ -217,6 +204,19 @@ hvantk runs Hail in **local Spark mode** — every `init_hail()` call uses local defaults (no master/memory config in the repo). On HPC that maps to **one exclusive node, many cores, high memory**, with Spark temp on **node-local scratch**. +> **`SPARK_LOCAL_DIRS` is mandatory when running from the container, and its absence +> is misdiagnosed.** Without a writable node-local scratch bound into the image, Hail +> init dies with +> `DiskBlockManager: ERROR: Failed to create any local dir`, followed by +> `Hail initialisation failed: [Errno 111] Connection refused` from py4j. The second +> message is what you see first and it reads like a network or proxy fault; it is not. +> `containers/hvantk_run.sh` sets the scratch dir up and binds it, so prefer: +> +> ```bash +> bash containers/hvantk_run.sh utils check-install +> HVANTK_BIND="$WORK:$WORK" bash containers/hvantk_run.sh reprocess clinvar:variants ... +> ``` + ```bash #!/bin/bash #SBATCH --job-name=hvantk-hail @@ -451,6 +451,13 @@ launching at scale. ## 10. Gotchas quick-reference - **Java 17 default** → Hail breaks. Force **Java 11** (baked into the container). +- **A bookworm base image** → there is no `openjdk-11` in Debian 12 at all; apt offers + `openjdk-17-jre-headless` instead. Use `python:3.10-slim-bullseye`. +- **`--fakeroot` with no `/etc/subuid` entry** → `no valid mapping entry found`, and an + unprivileged build is refused. Put a static `proot` on `PATH` instead (§3.3). +- **Container Hail init: `[Errno 111] Connection refused`** → almost never the network. + Look one line up for `DiskBlockManager: Failed to create any local dir`: Spark has no + writable scratch. Use `containers/hvantk_run.sh` (§4.1). - **`spark.driver.memory == --mem`** → silent cgroup OOM-kill. Use ~80%. - **`local[*]` on a shared node** → heartbeat timeouts. Pin to `--cpus-per-task`, or use `--exclusive`. @@ -463,5 +470,6 @@ launching at scale. - **Stray system/conda Python** below the 3.10 floor → use the container's Python; never run hvantk against an unmanaged interpreter. - **Core install ≠ full toolkit** — install the right **extras** (`hgc ptm - ancestry psroc constraint enrichex cohort viz duckdb`) in the image. + ancestry psroc constraint enrichex cohort viz duckdb expression`) in the image. + Dropping `expression` yields an image with no scanpy, so `hvantk expression …` fails. - **Scratch is purged** — copy results to project/home before the window. diff --git a/hvantk/core/plugin/api.py b/hvantk/core/plugin/api.py index 7368c233..9ab5f2d4 100644 --- a/hvantk/core/plugin/api.py +++ b/hvantk/core/plugin/api.py @@ -37,6 +37,13 @@ # than a fake ``sha256:...`` hash, so provenance never implies a real probe ran. STUB_FINGERPRINT_TOKEN = "stub:no-programmatic-source" +# Honest provenance token recorded by ``run_builder`` when the drift probe could +# not reach its source. A build must not be lost because provenance metadata was +# unobtainable -- the manual-acquisition plugins are built from staged files on +# nodes that may have no egress at all -- but the record must never imply a probe +# succeeded. Self-describing for the same reason as STUB_FINGERPRINT_TOKEN. +PROBE_UNAVAILABLE_TOKEN = "probe-unavailable:source-unreachable" + # Value a hand-seeded ``drift_fingerprint.json`` carries where a real checksum # belongs. A baseline holding it was written by hand, never captured from a live # probe, so it cannot equal any observed fingerprint. @@ -106,6 +113,40 @@ def stub_fingerprint(reason: str) -> dict: } +def normalize_etag(value: str | None) -> str | None: + """Reduce an ``ETag`` header to its bare entity-tag, or None if it carries none. + + ``str.strip('"')`` is wrong here and was shipped once: it strips a character + *set* from both ends, so ``W/"abc"`` becomes ``W/"abc`` (the leading ``W`` + blocks the left strip) and ``"abc"-gzip`` becomes ``abc"-gzip``. Both then get + recorded as a content signal, and because the drift bot regenerates drifted + baselines automatically, one such value bakes in permanently. + + Handles the three forms RFC 7232 and real servers produce: a strong tag + ``"abc"``, a weak tag ``W/"abc"``, and a transform-suffixed tag ``"abc"-gzip`` + (mod_deflate appends this when it compresses). Weak and suffixed forms reduce + to the same tag as the strong form, so a validator that flips between them on + a byte-identical object no longer reads as drift. + + Returns None for a missing, blank, or empty-quoted (``""``) tag, so callers can + fail closed on "no content signal" rather than recording an empty digest -- + which ``placeholder_baseline_reason`` would later flag as a hand-seeded + baseline, trapping the dataset in a probe_failed loop. + """ + if value is None: + return None + tag = value.strip() + if tag[:2].upper() == "W/": + tag = tag[2:] + # Drop a transform suffix appended after the closing quote. + if tag.startswith('"'): + closing = tag.find('"', 1) + if closing != -1: + tag = tag[: closing + 1] + tag = tag.strip('"').strip() + return tag or None + + class Builder(Protocol): """Phase B builder contract used by ``DatasetSpec.builder``. diff --git a/hvantk/core/plugin/run_builder.py b/hvantk/core/plugin/run_builder.py index b9ffd00e..ab0fb6b1 100644 --- a/hvantk/core/plugin/run_builder.py +++ b/hvantk/core/plugin/run_builder.py @@ -13,11 +13,19 @@ import hashlib import json +import logging from pathlib import Path from typing import Any from hvantk.core.models import BuildContext, Provenance -from hvantk.core.plugin.api import DatasetSpec, PROBE_FINGERPRINT_IGNORED_KEYS +from hvantk.core.plugin.api import ( + DatasetSpec, + DriftProbeError, + PROBE_FINGERPRINT_IGNORED_KEYS, + PROBE_UNAVAILABLE_TOKEN, +) + +logger = logging.getLogger(__name__) class BuilderContractError(RuntimeError): @@ -90,8 +98,32 @@ def run_builder_for_spec( f"migrate to Phase B contract before calling run_builder_for_spec()" ) - probe_result = spec.drift_probe() - fingerprint = _coerce_fingerprint(probe_result, spec.name) + # A probe failure must not destroy the build. The probe supplies provenance + # metadata, not build input, and every manual-acquisition plugin is built + # from a file staged by hand -- often on a compute node with no egress. Before + # the documentation-only plugins gained live probes their probes were pure + # in-process calls, so those builds needed no network at all; letting a + # DriftProbeError propagate here would have made `hvantk reprocess + # --skip-download --no-check-drift` fail offline, and `--no-check-drift` gates + # only the post-build check, not this call. + # + # The fallback is a self-describing token rather than a synthesized digest, so + # the provenance record never implies a probe ran (same reasoning as + # STUB_FINGERPRINT_TOKEN). + try: + probe_result = spec.drift_probe() + fingerprint = _coerce_fingerprint(probe_result, spec.name) + except DriftProbeError as exc: + logger.warning( + "%s: drift probe could not reach its source (%s); stamping provenance " + "with %r. The artifact is built normally; run `hvantk drift %s` once " + "connectivity is available to record a real fingerprint.", + spec.name, + exc, + PROBE_UNAVAILABLE_TOKEN, + spec.name, + ) + fingerprint = PROBE_UNAVAILABLE_TOKEN ctx = BuildContext( plugin=spec.name.split(":", 1)[0], diff --git a/hvantk/skills/alphagenome/SKILL.md b/hvantk/skills/alphagenome/SKILL.md index 42ef9107..b5c9da3e 100644 --- a/hvantk/skills/alphagenome/SKILL.md +++ b/hvantk/skills/alphagenome/SKILL.md @@ -24,8 +24,13 @@ with `chrom`/`pos`/`ref`/`alt` columns) and keys it by `(locus, alleles)`. A `config_path` param pointing to an AlphaGenome YAML config is required (see `tests/testdata/alphagenome_config.yaml`); `no_resume` (bool, default False) is optional. There is no built-in downloader (no `lifecycle.download` in -`plugin.yaml`). The drift probe (`drift_probe.py`, `fetch_fingerprint`) is a -placeholder stub. +`plugin.yaml`). The drift probe (`drift_probe.py`, `fetch_fingerprint`) reads the +published SDK release stream from PyPI's JSON API, which needs no credentials. + +**What it detects:** a new AlphaGenome SDK release, which is the signal to re-check +whether predictions still match a stored artifact. **What it cannot detect:** a +server-side model update shipped without an SDK release. No unauthenticated probe can +observe that, so the coverage claim stops there. ## Build invocation @@ -50,6 +55,7 @@ pytest hvantk/skills/alphagenome/tests No raw-data fixture is available for the alphagenome source (requires AlphaGenome API access). `tests/test_alphagenome.py` contains a registration-only test (`test_alphagenome_predictions_registered`) and a skipped round-trip test. The -only checked-in fixture is `tests/testdata/alphagenome_config.yaml`; the schema/row -snapshot and drift fingerprint paths declared in `plugin.yaml` are not yet -populated. +only checked-in fixture is `tests/testdata/alphagenome_config.yaml`. The drift +fingerprint (`tests/drift_fingerprint.json`) is populated from a live probe run; the +schema/row snapshot paths declared in `plugin.yaml` remain unpopulated, since a +credentialed live prediction API has no static artifact to snapshot. diff --git a/hvantk/skills/alphagenome/drift_probe.py b/hvantk/skills/alphagenome/drift_probe.py index 87d257d1..932ff964 100644 --- a/hvantk/skills/alphagenome/drift_probe.py +++ b/hvantk/skills/alphagenome/drift_probe.py @@ -1,20 +1,83 @@ -"""Drift probe for alphagenome — documentation-only source (stub). +"""alphagenome drift probe: SDK release check against the PyPI JSON API. -AlphaGenome is consumed as a live prediction API requiring credentials; there is -no static data file with a stable, programmatically-probeable URL to fingerprint. -This probe returns a structured stub sentinel so ``hvantk drift`` reports a -visible WARNING (status="stub") rather than a silent false-green. Replace with a -real probe if a direct data URL becomes available. See issue #177. +AlphaGenome is a credentialed live prediction service, so there is no static +artifact to fingerprint and issue #177 shipped a stub sentinel. For a live model +API, though, the meaningful upstream change is not a file but a *model or client +release*: predictions are generated on demand, so what makes a stored artifact +stale is the service behind it moving. The SDK is published openly on PyPI, whose +JSON API needs no credentials, so that release stream is directly probeable. + +Compared surface: the current version plus the sorted set of released versions. +Upload timestamps and file digests are excluded -- PyPI can re-host an unchanged +release, and the hgnc precedent is that a validator which moves without the +content changing produces nothing but no-op pull requests. + +What this detects: a new AlphaGenome SDK release, which is the signal to re-check +whether predictions still match a stored artifact. What it cannot detect: a +server-side model update shipped without an SDK release, which no unauthenticated +probe can see. That limit is a property of the service and is recorded in +SKILL.md so the coverage claim stays honest. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.core.utils.http import request_with_retry -_REASON = ( - "AlphaGenome is a live prediction API requiring credentials; " - "no static data file to fingerprint" -) +PROBE_VERSION = 2 +ALPHAGENOME_PYPI_URL = "https://pypi.org/pypi/alphagenome/json" + +_FILENAME = "alphagenome-sdk-releases" +_TIMEOUT_S = (5.0, 15.0) def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Fingerprint the published AlphaGenome SDK release set.""" + try: + resp = request_with_retry( + "GET", ALPHAGENOME_PYPI_URL, timeout=_TIMEOUT_S, allow_redirects=True + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure: {exc}") from exc + + # Parsed in its own block: requests' JSONDecodeError subclasses both + # ValueError and RequestException, so decoding inside the block above would + # report a malformed body as an HTTP failure. + try: + payload = resp.json() + except ValueError as exc: + raise DriftProbeError(f"PyPI returned non-JSON: {exc}") from exc + + current = (payload.get("info") or {}).get("version") + # Fail closed on the field that actually carries the signal. `info.version` is + # the supported one; `releases` is deprecated on this endpoint and slated for + # removal, so requiring it would turn a PyPI API change into a permanent + # probe_failed for an SDK that never moved. + if not current: + raise DriftProbeError( + "PyPI returned no info.version for alphagenome; the project or the " + "API shape has probably changed." + ) + + compared: dict[str, object] = {"current_version": current} + releases = sorted((payload.get("releases") or {}).keys()) + if releases: + compared["release_count"] = len(releases) + + return { + "probe_version": PROBE_VERSION, + "source_version": current, + # The version IS the signal; a sha256 over it would be a pure function of + # a value already in the compared surface. The full release list is + # deliberately NOT compared: it is deprecated upstream, and it grows on + # pre-release and yanked uploads that no build would ever install. + "headers": {_FILENAME: compared}, + "checksums": {}, + "informational": {"releases_found": releases}, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/alphagenome/tests/drift_fingerprint.json b/hvantk/skills/alphagenome/tests/drift_fingerprint.json new file mode 100644 index 00000000..d2fd4b81 --- /dev/null +++ b/hvantk/skills/alphagenome/tests/drift_fingerprint.json @@ -0,0 +1,28 @@ +{ + "probe_version": 2, + "source_version": "0.8.0", + "headers": { + "alphagenome-sdk-releases": { + "current_version": "0.8.0", + "release_count": 12 + } + }, + "checksums": {}, + "informational": { + "releases_found": [ + "0.0.1", + "0.0.2", + "0.1.0", + "0.2.0", + "0.3.0", + "0.4.0", + "0.5.0", + "0.5.1", + "0.6.0", + "0.6.1", + "0.7.0", + "0.8.0" + ] + }, + "fetched_at": "2026-09-01T07:32:19.398674+00:00" +} diff --git a/hvantk/skills/alphagenome/tests/test_drift_probe.py b/hvantk/skills/alphagenome/tests/test_drift_probe.py new file mode 100644 index 00000000..b5b4bee0 --- /dev/null +++ b/hvantk/skills/alphagenome/tests/test_drift_probe.py @@ -0,0 +1,76 @@ +"""alphagenome drift probe should fingerprint the published SDK release set. + +Runs OFFLINE via requests_mock. The prediction service itself stays credentialed +(issue #177 was right that there is no static artifact); what this probe reaches +is the openly published SDK release stream. +""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.alphagenome.drift_probe import ( + ALPHAGENOME_PYPI_URL, + fetch_fingerprint, +) + + +def _payload(current="0.8.0", releases=("0.7.0", "0.8.0")): + return { + "info": {"name": "alphagenome", "version": current}, + "releases": {v: [] for v in releases}, + } + + +def test_fetch_fingerprint_shape(): + with requests_mock.Mocker() as m: + m.get(ALPHAGENOME_PYPI_URL, json=_payload()) + fp = fetch_fingerprint() + + assert fp["source_version"] == "0.8.0" + assert fp["informational"]["releases_found"] == ["0.7.0", "0.8.0"] + + +def test_new_sdk_release_moves_the_checksum(): + with requests_mock.Mocker() as m: + m.get(ALPHAGENOME_PYPI_URL, json=_payload()) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get( + ALPHAGENOME_PYPI_URL, + json=_payload(current="0.9.0", releases=("0.7.0", "0.8.0", "0.9.0")), + ) + after = fetch_fingerprint() + + assert before["headers"] != after["headers"] + assert after["source_version"] == "0.9.0" + + +def test_missing_version_fails_closed(): + """Discriminating case: `releases` present, `info.version` absent.""" + with requests_mock.Mocker() as m: + m.get(ALPHAGENOME_PYPI_URL, json={"info": {}, "releases": {"0.8.0": []}}) + with pytest.raises(DriftProbeError, match="no info.version"): + fetch_fingerprint() + + +def test_deprecated_releases_key_absent_still_probes(): + """PyPI deprecated `releases` on this endpoint. Requiring it would turn an + upstream API change into a permanent probe_failed for an SDK that never moved, + while `info.version` -- the field carrying the signal -- is unaffected.""" + with requests_mock.Mocker() as m: + m.get(ALPHAGENOME_PYPI_URL, json={"info": {"version": "0.8.0"}}) + fp = fetch_fingerprint() + + assert fp["source_version"] == "0.8.0" + assert fp["headers"]["alphagenome-sdk-releases"]["current_version"] == "0.8.0" + + +def test_non_json_fails_closed(): + with requests_mock.Mocker() as m: + m.get(ALPHAGENOME_PYPI_URL, text="not json") + with pytest.raises(DriftProbeError, match="non-JSON"): + fetch_fingerprint() diff --git a/hvantk/skills/cosmic_cgc/SKILL.md b/hvantk/skills/cosmic_cgc/SKILL.md index 2331fe2f..73122809 100644 --- a/hvantk/skills/cosmic_cgc/SKILL.md +++ b/hvantk/skills/cosmic_cgc/SKILL.md @@ -12,6 +12,23 @@ Upstream: https://cancer.sanger.ac.uk/census - `cosmic-cgc:submissions` — gene-level cancer gene census table, keyed by `gene_symbol` by default (or `hgnc_id` if a `gene_catalog` is provided) +## Drift detection + +The drift probe (`drift_probe.py`, `fetch_fingerprint`) reads the per-release anchors +(`id="v"`) from COSMIC's public release-notes page. The Census *data* stays +login- and licence-gated -- `cancer.sanger.ac.uk/census` answers 302 to +`/cosmic/login` -- so acquisition remains manual and no data URL is probed. + +**What it detects:** a new COSMIC release. **What it cannot detect:** a change to the +Census contents within a release; no unauthenticated probe can see that. + +Two details are load-bearing. The trailing slash matters -- `/cosmic/release_notes` +returns 200 while `/cosmic/release_notes/` redirects to the login form -- and the probe +rejects any redirected response rather than scraping a login page. And it anchors on the +`id="v"` attributes, never on prose: matching `COSMIC v` in body text picked up +`v16`/`v18`/`v20` from sentences about the *Actionability* product, a different version +series, so an unrelated editorial edit would have opened a no-op pull request. + ## Build ```bash diff --git a/hvantk/skills/cosmic_cgc/drift_probe.py b/hvantk/skills/cosmic_cgc/drift_probe.py index 15ef7e5a..d061f5c1 100644 --- a/hvantk/skills/cosmic_cgc/drift_probe.py +++ b/hvantk/skills/cosmic_cgc/drift_probe.py @@ -1,20 +1,88 @@ -"""Drift probe for cosmic-cgc — documentation-only source (stub). +"""cosmic-cgc drift probe: release index from the public release notes. -The COSMIC Cancer Gene Census is behind a login/license gate -(cancer.sanger.ac.uk/census); there is no public direct URL to fingerprint. -This probe returns a structured stub sentinel so ``hvantk drift`` reports a -visible WARNING (status="stub") rather than a silent false-green. Replace with a -real probe if a direct data URL becomes available. See issue #177. +The Cancer Gene Census *data* is login- and licence-gated, which issue #177 +recorded correctly: ``cancer.sanger.ac.uk/census`` answers 302 to +``/cosmic/login``, so no direct data URL exists and acquisition stays manual. +The conclusion that nothing could be probed does not follow, though. COSMIC +publishes its release notes without a login, and those carry a per-release +anchor, so a release roll-over is detectable even though the archive is not. + +Note the trailing slash matters: ``/cosmic/release_notes`` returns 200 while +``/cosmic/release_notes/`` redirects to the login page. + +**Anchor on the id attributes, never on prose.** A first version matched +``COSMIC\\s+v(\\d+)`` anywhere in the body and produced +``[v16, v18, v20, v101, v102, v103, v104]`` -- a non-contiguous set that is not a +release index at all. v16/v18/v20 come from sentences about the *Actionability* +product ("COSMIC v20 of the Actionability data"), a different product line with +its own version series. Any editorial sentence naming an old release would have +entered the compared surface and opened a no-op pull request, and a +forward-looking "coming in COSMIC v105" would have reported a release that did +not exist. The page instead carries ``id="v101"`` ... ``id="v104"`` anchors, one +per real release, which is what this probe reads. + +What this detects: a new COSMIC release. What it cannot detect: a change to the +Census contents within a release, which no unauthenticated probe can see. That +limit is a property of the licence gate and is recorded in SKILL.md. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +import re +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.core.utils.http import request_with_retry -_REASON = ( - "COSMIC Cancer Gene Census is login/license-gated " - "(cancer.sanger.ac.uk/census); no public direct URL to fingerprint" -) +PROBE_VERSION = 2 +COSMIC_RELEASE_NOTES_URL = "https://cancer.sanger.ac.uk/cosmic/release_notes" + +# Per-release anchors, e.g. id="v104". Deliberately NOT a prose pattern. +_RELEASE_ANCHOR_RE = re.compile(r'id="v(\d+)"', re.IGNORECASE) + +_FILENAME = "cosmic-release-index" +_TIMEOUT_S = (5.0, 15.0) def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Fingerprint the release index on the public COSMIC release notes.""" + try: + resp = request_with_retry( + "GET", COSMIC_RELEASE_NOTES_URL, timeout=_TIMEOUT_S, allow_redirects=True + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure: {exc}") from exc + + # The host is known to redirect to a login form (the trailing-slash path does + # exactly that), and a login page answers 200. Without this a redirect would + # be scraped as if it were the release notes. + if resp.history: + raise DriftProbeError( + f"COSMIC release notes redirected to {resp.url!r}; the page has " + "probably moved or now requires a login." + ) + + # Decoded explicitly: requests falls back to ISO-8859-1 for text/html with no + # charset, which would mangle a non-breaking space and silently change what + # the pattern matches. + body = resp.content.decode("utf-8", errors="replace") + versions = sorted({int(m.group(1)) for m in _RELEASE_ANCHOR_RE.finditer(body)}) + if not versions: + raise DriftProbeError( + "COSMIC release notes carried no 'id=\"v\"' release anchors; the " + "page layout has probably changed." + ) + + return { + "probe_version": PROBE_VERSION, + "source_version": f"v{max(versions)}", + # The release list IS the signal; a sha256 over it would be a pure + # function of a value already in the compared surface and would add no + # detection power. + "headers": {_FILENAME: [f"v{v}" for v in versions]}, + "checksums": {}, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/cosmic_cgc/tests/drift_fingerprint.json b/hvantk/skills/cosmic_cgc/tests/drift_fingerprint.json new file mode 100644 index 00000000..92f06ad4 --- /dev/null +++ b/hvantk/skills/cosmic_cgc/tests/drift_fingerprint.json @@ -0,0 +1,14 @@ +{ + "probe_version": 2, + "source_version": "v104", + "headers": { + "cosmic-release-index": [ + "v101", + "v102", + "v103", + "v104" + ] + }, + "checksums": {}, + "fetched_at": "2026-09-01T07:32:18.934283+00:00" +} diff --git a/hvantk/skills/cosmic_cgc/tests/test_drift_probe.py b/hvantk/skills/cosmic_cgc/tests/test_drift_probe.py new file mode 100644 index 00000000..200d0582 --- /dev/null +++ b/hvantk/skills/cosmic_cgc/tests/test_drift_probe.py @@ -0,0 +1,107 @@ +"""cosmic-cgc drift probe should read the release index, never page prose.""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.cosmic_cgc.drift_probe import ( + COSMIC_RELEASE_NOTES_URL, + fetch_fingerprint, +) + +# Anchors are the real index; the prose deliberately names OTHER product versions, +# which is what the live page does ("COSMIC v20 of the Actionability data"). +_PAGE = """ + +

COSMIC v104

+

COSMIC v104 is combined with COSMIC v20 of the Actionability data.

+

COSMIC v103

+

COSMIC v18 of the Actionability data are released.

+

COSMIC v102

+ +""" + + +def test_prose_versions_are_excluded_from_the_index(): + """v20/v18 belong to the Actionability product, a different version series. + Matching them produced a non-contiguous 'release list' and would open a no-op + PR on any editorial edit naming an old release.""" + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text=_PAGE) + fp = fetch_fingerprint() + + found = fp["headers"]["cosmic-release-index"] + assert found == ["v102", "v103", "v104"] + assert "v20" not in found and "v18" not in found + + +def test_editorial_prose_edit_does_not_move_the_signal(): + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text=_PAGE) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get( + COSMIC_RELEASE_NOTES_URL, + text=_PAGE.replace("", "

Unchanged since COSMIC v95.

"), + ) + after = fetch_fingerprint() + + assert before["headers"] == after["headers"] + + +def test_forward_looking_prose_cannot_bump_source_version(): + """'coming in COSMIC v105' must not report a release that does not exist.""" + with requests_mock.Mocker() as m: + m.get( + COSMIC_RELEASE_NOTES_URL, + text=_PAGE.replace("", "

Coming soon: COSMIC v105.

"), + ) + fp = fetch_fingerprint() + + assert fp["source_version"] == "v104" + + +def test_versions_sort_numerically(): + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text='') + fp = fetch_fingerprint() + + assert fp["source_version"] == "v104" + assert fp["headers"]["cosmic-release-index"] == ["v99", "v104"] + + +def test_new_release_moves_the_signal(): + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text=_PAGE) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text=_PAGE + '

COSMIC v105

') + after = fetch_fingerprint() + + assert before["headers"] != after["headers"] + assert after["source_version"] == "v105" + + +def test_login_redirect_fails_closed(): + """An actual redirect, not just a login-shaped body: the host is known to 302 + the trailing-slash path to /cosmic/login, and a login page answers 200.""" + with requests_mock.Mocker() as m: + m.get( + COSMIC_RELEASE_NOTES_URL, + status_code=302, + headers={"Location": "https://cancer.sanger.ac.uk/cosmic/login"}, + ) + m.get("https://cancer.sanger.ac.uk/cosmic/login", text="

Please log in

") + with pytest.raises(DriftProbeError, match="redirected"): + fetch_fingerprint() + + +def test_missing_anchors_fail_closed(): + with requests_mock.Mocker() as m: + m.get(COSMIC_RELEASE_NOTES_URL, text="Please log in") + with pytest.raises(DriftProbeError, match="no 'id="): + fetch_fingerprint() diff --git a/hvantk/skills/dbnsfp/SKILL.md b/hvantk/skills/dbnsfp/SKILL.md index fe0d5868..0a7c15b5 100644 --- a/hvantk/skills/dbnsfp/SKILL.md +++ b/hvantk/skills/dbnsfp/SKILL.md @@ -51,7 +51,21 @@ column with no entry is simply unknown, which is the safe default. ## Notes -The drift probe (`drift_probe.fetch_fingerprint`) is a stub; a real probe +The drift probe (`drift_probe.fetch_fingerprint`) scrapes the release list the +upstream landing page advertises. + +**What it detects:** a new dbNSFP release being advertised. **What it cannot detect:** +an in-place change to an archive's contents, or the download links being repaired -- +the markup names the same archives either way. It deliberately never hashes the page +body: Google Sites re-renders per request (352,830 vs 352,716 bytes on two consecutive +fetches), so hashing it would flag drift on every run. + +> **The documented download path is broken.** Every `dbNSFP*.zip` the landing page links +> returns 404 -- the S3 bucket answers `NoSuchBucket` -- and the `database.liulab.science` +> mirror does not resolve. Tracked as issue #321. Acquisition currently has no working +> public route. + +A real downloader should be implemented in a follow-up. No downloader is wired in `plugin.yaml` lifecycle yet; upstream files are expected to be externally materialized for now. diff --git a/hvantk/skills/dbnsfp/drift_probe.py b/hvantk/skills/dbnsfp/drift_probe.py index aae7be88..23bd3f7d 100644 --- a/hvantk/skills/dbnsfp/drift_probe.py +++ b/hvantk/skills/dbnsfp/drift_probe.py @@ -1,20 +1,121 @@ -"""Drift probe for dbnsfp — documentation-only source (stub). +"""dbnsfp drift probe: release-list scrape of the upstream landing page. -dbNSFP is distributed from a landing page (sites.google.com/site/jpopgen/dbNSFP) -with no stable direct data URL to fingerprint. This probe returns a structured -stub sentinel so ``hvantk drift`` reports a visible WARNING (status="stub") -rather than a silent false-green. Replace with a real probe if a direct data URL -becomes available. See issue #177. +dbNSFP has no reachable direct data URL, which issue #177 recorded correctly -- +but the conclusion that nothing could be probed does not follow. The landing page +itself is fetchable and advertises the release set, so a release roll-over is +detectable even though the archives are not. + +Why not the archives: the landing page links every release at +``https://dbnsfp.s3.amazonaws.com/dbNSFP.zip``, and every one of those +is dead -- the bucket answers ``NoSuchBucket``, so the links 404 rather than +merely being gated. The alternative mirror the page names +(``database.liulab.science``) does not resolve at all. Acquisition therefore +stays manual, and this probe reports on the *advertised release set* rather than +on any data file. That the documented download path is broken is tracked +separately as issue #321; it is a documentation defect, not a drift signal. + +**The raw page must never be hashed.** Google Sites re-renders per request: two +consecutive fetches returned 352,830 and 352,716 bytes. Hashing the body would +flag drift on every probe run and open a nightly no-op pull request. The +extracted release list was identical across those same two fetches, so the probe +compares that projection instead. This is the msigdb index-page pattern, and here +it is a correctness requirement rather than a stylistic choice. + +What this detects: a new dbNSFP release being advertised. What it cannot detect: +an in-place change to an archive's contents, or the download links being +repaired -- the page markup names the same archives either way, so a restored +bucket would not move this fingerprint. Those limits are recorded in SKILL.md. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +import re +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.core.utils.http import request_with_retry + +PROBE_VERSION = 2 +DBNSFP_LANDING_URL = "https://sites.google.com/site/jpopgen/dbNSFP" + +# Matches the release archives the page advertises. The optional `v` is +# load-bearing: dbNSFP's 2.x and 3.x generations are published as `dbNSFPv3.5a.zip`, +# and a pattern requiring a digit straight after "dbNSFP" silently matched none of +# them -- so a next release named `dbNSFPv5.0a.zip` would have left the projection +# unchanged and reported clean on a real roll-over. +_RELEASE_REGEX = re.compile(r"dbNSFPv?(\d[\w.]*?)\.zip", re.IGNORECASE) -_REASON = ( - "dbNSFP is distributed from a landing page " - "(sites.google.com/site/jpopgen/dbNSFP); no stable direct data URL" -) +# Academic releases carry the "a" suffix ("c" is the commercial build); hvantk +# builds from the academic one. Case-insensitive to match _RELEASE_REGEX, which +# would otherwise capture `4.9A` into the set while this rejected it, leaving a +# phantom release in the list and a regressed source_version. +_ACADEMIC_REGEX = re.compile(r"^(\d+(?:\.\d+)*)a$", re.IGNORECASE) + +_FILENAME = "dbNSFP-release-index" +_TIMEOUT_S = (5.0, 15.0) + + +def _latest_academic(versions: list[str]) -> str | None: + """Highest academic release, compared componentwise rather than as text.""" + parsed = [] + for v in versions: + m = _ACADEMIC_REGEX.match(v) + if m: + parts = tuple(int(p) for p in m.group(1).split(".")) + parsed.append((parts, v)) + if not parsed: + return None + # Componentwise so 4.10a beats 4.9a; a text sort would not. + return max(parsed)[1] def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Fingerprint the release set advertised on the dbNSFP landing page.""" + try: + resp = request_with_retry( + "GET", DBNSFP_LANDING_URL, timeout=_TIMEOUT_S, allow_redirects=True + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure: {exc}") from exc + + # Decoded explicitly: requests falls back to ISO-8859-1 for text/html with no + # charset, and its chardet/charset_normalizer fallback is environment + # dependent, so the same page could otherwise yield two different projections + # on two machines. + body = resp.content.decode("utf-8", errors="replace") + + # Lowercased before entering the set so a cosmetic recasing of one link + # cannot present as an extra release. + versions = sorted({m.group(1).lower() for m in _RELEASE_REGEX.finditer(body)}) + # Fail closed. An empty match set means the page moved or its markup changed, + # not that dbNSFP shipped zero releases; recording it would bake an empty + # baseline that every later run compares equal to. + if not versions: + raise DriftProbeError( + "dbNSFP landing page advertised no dbNSFP.zip releases; " + "the page layout has probably changed." + ) + + latest = _latest_academic(versions) + # Also fail closed here. Silently recording source_version: null would let the + # bot commit that null as the baseline, after which the probe reports clean + # forever having quietly stopped identifying a release at all. + if latest is None: + raise DriftProbeError( + f"dbNSFP advertised {len(versions)} releases but none matched the " + "academic 'a' naming; the release scheme has probably " + "changed." + ) + + return { + "probe_version": PROBE_VERSION, + "source_version": latest, + # The release list IS the signal; a sha256 over it would be a pure + # function of a value already in the compared surface. + "headers": {_FILENAME: versions}, + "checksums": {}, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/dbnsfp/tests/drift_fingerprint.json b/hvantk/skills/dbnsfp/tests/drift_fingerprint.json new file mode 100644 index 00000000..44874aaa --- /dev/null +++ b/hvantk/skills/dbnsfp/tests/drift_fingerprint.json @@ -0,0 +1,69 @@ +{ + "probe_version": 2, + "source_version": "4.9a", + "headers": { + "dbNSFP-release-index": [ + "1.2", + "1.3", + "2.0", + "2.0b1_variant", + "2.0b2", + "2.0b3", + "2.0b4", + "2.1", + "2.2", + "2.3", + "2.4", + "2.5", + "2.6", + "2.7", + "2.8", + "2.9", + "2.9.1", + "2.9.2", + "2.9.3", + "3.0a", + "3.0b1a", + "3.0b1c", + "3.0b2a", + "3.0b2c", + "3.0c", + "3.1a", + "3.1c", + "3.2a", + "3.2c", + "3.3a", + "3.3c", + "3.4a", + "3.4c", + "3.5a", + "3.5c", + "4.0a", + "4.0b1a", + "4.0b1c", + "4.0b2a", + "4.0b2c", + "4.0c", + "4.1a", + "4.1c", + "4.2a", + "4.2c", + "4.3a", + "4.3c", + "4.4a", + "4.4c", + "4.5a", + "4.5c", + "4.6a", + "4.6c", + "4.7a", + "4.7c", + "4.8a", + "4.8c", + "4.9a", + "4.9c" + ] + }, + "checksums": {}, + "fetched_at": "2026-09-01T07:32:18.168305+00:00" +} diff --git a/hvantk/skills/dbnsfp/tests/test_drift_probe.py b/hvantk/skills/dbnsfp/tests/test_drift_probe.py new file mode 100644 index 00000000..cd58518a --- /dev/null +++ b/hvantk/skills/dbnsfp/tests/test_drift_probe.py @@ -0,0 +1,113 @@ +"""dbnsfp drift probe should fingerprint the advertised release list, not the page.""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.dbnsfp.drift_probe import DBNSFP_LANDING_URL, fetch_fingerprint + +_LINKS = [ + "dbNSFP4.8a.zip", "dbNSFP4.8c.zip", "dbNSFP4.9a.zip", "dbNSFP4.9c.zip", + "dbNSFPv3.5a.zip", "dbNSFPv2.9.3.zip", +] +_PAGE = "" + "".join( + f'{n}' for n in _LINKS +) + "" + + +def _page(links): + return "" + "".join( + f'{n}' for n in links + ) + "" + + +def test_v_prefixed_releases_are_captured(): + """dbNSFP's 2.x and 3.x generations publish as `dbNSFPv3.5a.zip`. A pattern + requiring a digit straight after "dbNSFP" matched none of them, so a next + release named `dbNSFPv5.0a.zip` would have left the projection unchanged and + reported clean on a real roll-over.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_PAGE) + fp = fetch_fingerprint() + + found = fp["headers"]["dbNSFP-release-index"] + assert "3.5a" in found + assert "2.9.3" in found + + +def test_reports_latest_academic_release(): + """`a` is the academic build hvantk uses; `c` is commercial and must not win.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_PAGE) + fp = fetch_fingerprint() + + assert fp["source_version"] == "4.9a" + + +def test_versions_compare_componentwise_not_as_text(): + """4.10a must beat 4.9a. The committed baseline already sits at 4.9a, so the + very next minor release is the one a text sort gets wrong.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_page(["dbNSFP4.9a.zip", "dbNSFP4.10a.zip"])) + fp = fetch_fingerprint() + + assert fp["source_version"] == "4.10a" + + +def test_projection_ignores_markup_and_link_order(): + """The page re-renders per request (352,830 vs 352,716 bytes on two consecutive + live fetches), so the compared value must depend only on the release SET.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_PAGE) + first = fetch_fingerprint() + + reordered = _page(list(reversed(_LINKS))).replace( + "", "

unrelated editorial edit

" + ) + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=reordered) + second = fetch_fingerprint() + + assert first["headers"] == second["headers"] + + +def test_case_variation_does_not_invent_a_release(): + """The extraction pattern is case-insensitive, so an uppercase link must fold + onto the same release rather than entering the set twice.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_page(["dbNSFP4.9a.zip", "dbNSFP4.9A.zip"])) + fp = fetch_fingerprint() + + assert fp["headers"]["dbNSFP-release-index"] == ["4.9a"] + assert fp["source_version"] == "4.9a" + + +def test_new_release_moves_the_signal(): + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_PAGE) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_page(_LINKS + ["dbNSFP5.0a.zip"])) + after = fetch_fingerprint() + + assert before["headers"] != after["headers"] + assert after["source_version"] == "5.0a" + + +def test_empty_release_list_fails_closed(): + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text="no releases here") + with pytest.raises(DriftProbeError, match="advertised no"): + fetch_fingerprint() + + +def test_no_academic_release_fails_closed(): + """Silently recording source_version: null would let the bot commit that null, + after which the probe reports clean having stopped identifying a release.""" + with requests_mock.Mocker() as m: + m.get(DBNSFP_LANDING_URL, text=_page(["dbNSFP4.9c.zip"])) + with pytest.raises(DriftProbeError, match="academic"): + fetch_fingerprint() diff --git a/hvantk/skills/gevir/SKILL.md b/hvantk/skills/gevir/SKILL.md index 8d1d6ffb..24b75057 100644 --- a/hvantk/skills/gevir/SKILL.md +++ b/hvantk/skills/gevir/SKILL.md @@ -7,8 +7,14 @@ Genetics* 52(1):35-39; DOI 10.1038/s41588-019-0560-2) and ranks **19,361** protein-coding genes by their intolerance to variation, derived from the density and spatial distribution of protein-coding variants observed across ~138,632 gnomAD exome and genome sequences. GeVIR is a gene-level metric — it is **not** a -variant-level pathogenicity score. Upstream code and data are at -https://github.com/gevirank/gevir. +variant-level pathogenicity score. + +The metric table is distributed as **Supplementary Table 2** of that paper, served +from Springer's static-content CDN as the article's MOESM3 object (an `.xlsx` +workbook, ~10.3 MB; only that one of the six MOESM slots is public). The authors' +repository at https://github.com/gevirank/gevir ships the **analysis code only** -- +its `tables/` directory holds a placeholder file -- so it is not a source for the +table and cannot be used as a drift target. ## Dataset @@ -35,13 +41,24 @@ Optional plugin args (e.g. field selection) can be passed with ## Notes -The drift probe (`hvantk/skills/gevir/drift_probe.py`, `fetch_fingerprint`) is a -documentation-only stub: GeVIR is published as supplementary data, so there is no -programmatic data URL to fingerprint and `hvantk drift` reports status="stub". -No downloader is implemented yet; the GeVIR table is small (~1-2 MB), public, and -served from a stable URL, so it qualifies for a real downloader under the -project's downloader framework — a recommended follow-up. Until then, upstream -files are expected to be materialized externally. +The drift probe (`hvantk/skills/gevir/drift_probe.py`, `fetch_fingerprint`) issues +a single HEAD against the article's supplementary object (§ above) and compares +its content-hash ETag plus Content-Length; the workbook body is never +transferred. It replaced a documentation-only stub once that URL was confirmed +addressable (issue #177, which had recorded the source as publication-only and +therefore unprobeable). + +**The probe target and the build input are different files.** The probe watches +the upstream `.xlsx` (~10.3 MB); the builder reads a bgzipped TSV +(`gevir_metrics_pmid31873297.tsv.bgz`, ~1-2 MB) derived from sheet `table_2` of +that workbook. The catalog `files` entry describes the derived TSV, which is why +it carries no download URL: fetching the upstream URL does not yield that file +without an extract-and-convert step. + +No downloader is implemented yet. A real one would have to extract sheet +`table_2` and BGZF-compress it, not just fetch the URL, so it is more than the +usual thin wrapper — a recommended follow-up. Until then, upstream files are +expected to be materialized externally. ## Schema diff --git a/hvantk/skills/gevir/drift_probe.py b/hvantk/skills/gevir/drift_probe.py index 3172147e..6c9f5fa7 100644 --- a/hvantk/skills/gevir/drift_probe.py +++ b/hvantk/skills/gevir/drift_probe.py @@ -1,20 +1,111 @@ -"""Drift probe for gevir — documentation-only source (stub). +"""GeVIR drift probe: HEAD against the published supplementary object. -GeVIR metrics are published as supplementary data (PMID 31873297); there is no -programmatic data URL to fingerprint. This probe returns a structured stub -sentinel so ``hvantk drift`` reports a visible WARNING (status="stub") rather -than a silent false-green. Replace with a real probe if a direct data URL becomes -available. See issue #177. +GeVIR metrics are distributed as supplementary data to the Nature Genetics paper +(PMID 31873297, DOI 10.1038/s41588-019-0560-2), served from Springer's +static-content CDN at a stable, direct URL. Issue #177 recorded this source as +having "no probeable URL at all (publication PMID 31873297 only)" and shipped a +stub sentinel; that conflated *publication-only distribution* with *no +addressable URL*. The ESM object below answers a HEAD with both Content-Length +and a content-hash ETag, so an hgnc-style probe is feasible after all. + +The authors' code repository (github.com/gevirank/gevir) is NOT a usable source: +it ships the analysis code, and its ``tables/`` directory holds only a +placeholder file. + +Of the six MOESM slots for this article only MOESM3 is public (the rest answer +403). It is an Excel workbook, not the bgzipped TSV the builder consumes -- sheet +``table_2`` has to be extracted and converted first -- so this probe watches the +*published upstream*, while the local build input is materialized externally. + +Compared surface: the ETag and Content-Length, both under ``headers``. Springer +serves the ETag as an MD5 over the object body, so unlike a size+mtime validator +it is a true content digest. They live under ``headers`` rather than ``checksums`` +because the drift bot reads ``checksums`` as a hash of the column-header row -- +a schema signal -- and this probe never fetches a body, so it has no schema +signal to offer; recording one there tiered every routine content update as a +schema change. peptideatlas ships this same shape: validator metadata under +``headers``, ``checksums`` left empty. + +``Last-Modified`` is demoted to ``informational`` following the hgnc precedent, +where 8 of 8 drift PRs moved only the timestamp. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError, normalize_etag + +PROBE_VERSION = 2 -_REASON = ( - "GeVIR metrics are published as supplementary data (PMID 31873297); " - "no programmatic data URL to fingerprint" +# Supplementary Tables workbook for Abramovs et al., Nat Genet 2020. +GEVIR_SUPPLEMENTARY_URL = ( + "https://static-content.springer.com/esm/" + "art%3A10.1038%2Fs41588-019-0560-2/MediaObjects/41588_2019_560_MOESM3_ESM.xlsx" ) +_FILENAME = "41588_2019_560_MOESM3_ESM.xlsx" +_TIMEOUT_S = 30 + +# Pinned so Content-Length always describes the stored object. A server that +# compresses on the fly omits Content-Length and appends a transform suffix to +# the ETag; Springer does not today, but the fingerprint must not silently change +# meaning if that ever turns on. +_HEADERS = {"Accept-Encoding": "identity"} + def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Lightweight fingerprint of the live GeVIR supplementary object. + + Issues a single HEAD; the 10 MB workbook is never transferred. + """ + try: + head = requests.head( + GEVIR_SUPPLEMENTARY_URL, + timeout=_TIMEOUT_S, + allow_redirects=True, + headers=_HEADERS, + ) + head.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure: {exc}") from exc + + content_length = head.headers.get("Content-Length") + etag = normalize_etag(head.headers.get("ETag")) + + # Fail closed on emptiness, not just absence. Requiring BOTH validators is + # deliberate: accepting an ETag-only response would record + # `content_length: None`, and because the drift bot regenerates drifted + # baselines automatically, that null bakes in and every later normal response + # reads as drift. An empty `ETag: ""` is likewise rejected rather than stored, + # since placeholder_baseline_reason would then read the baseline as + # hand-seeded and trap the dataset in a probe_failed loop that regenerating + # cannot clear. + if not content_length or not etag: + raise DriftProbeError( + f"GeVIR response carried no usable content signal " + f"(Content-Length={content_length!r}, ETag={etag!r}); refusing to " + "record a fingerprint that would compare equal to its baseline " + "forever." + ) + + # Asserted request-side above; verified response-side here, because a caching + # proxy may compress regardless and the recorded length would then describe + # the compressed body rather than the object. + encoding = (head.headers.get("Content-Encoding") or "identity").lower() + if encoding != "identity": + raise DriftProbeError( + f"GeVIR response was {encoding}-encoded despite an identity request; " + "Content-Length would describe the compressed body." + ) + + return { + "probe_version": PROBE_VERSION, + "source_version": None, + "headers": {_FILENAME: {"content_length": content_length, "etag": etag}}, + "checksums": {}, + "informational": {"last_modified": head.headers.get("Last-Modified")}, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/gevir/tests/drift_fingerprint.json b/hvantk/skills/gevir/tests/drift_fingerprint.json index 68a814f5..5375eb26 100644 --- a/hvantk/skills/gevir/tests/drift_fingerprint.json +++ b/hvantk/skills/gevir/tests/drift_fingerprint.json @@ -1,5 +1,15 @@ { - "probe_status": "stub", - "reason": "GeVIR metrics are published as supplementary data (PMID 31873297); no programmatic data URL to fingerprint", - "fingerprint": "stub:no-programmatic-source" + "probe_version": 2, + "source_version": null, + "headers": { + "41588_2019_560_MOESM3_ESM.xlsx": { + "content_length": "10270511", + "etag": "6423adf134a669acc357f619d1162009" + } + }, + "checksums": {}, + "informational": { + "last_modified": "Tue, 14 Nov 2023 18:23:50 GMT" + }, + "fetched_at": "2026-09-01T00:41:41.211380+00:00" } diff --git a/hvantk/skills/gevir/tests/test_drift_probe.py b/hvantk/skills/gevir/tests/test_drift_probe.py new file mode 100644 index 00000000..548f0060 --- /dev/null +++ b/hvantk/skills/gevir/tests/test_drift_probe.py @@ -0,0 +1,130 @@ +"""GeVIR drift probe should fingerprint the supplementary object offline. + +Runs OFFLINE via requests_mock, so CI never hits Springer. The probe replaced a +stub sentinel (issue #177) once the article's ESM object was confirmed to answer +a HEAD with both Content-Length and a content-hash ETag. +""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.gevir.drift_probe import GEVIR_SUPPLEMENTARY_URL, fetch_fingerprint + +_FILENAME = "41588_2019_560_MOESM3_ESM.xlsx" +_RESPONSE_HEADERS = { + "Content-Length": "10270511", + "ETag": '"6423adf134a669acc357f619d1162009"', + "Last-Modified": "Tue, 14 Nov 2023 18:23:50 GMT", +} + + +def test_fetch_fingerprint_shape(): + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers=_RESPONSE_HEADERS) + fp = fetch_fingerprint() + + assert fp["probe_version"] == 2 + assert fp["headers"][_FILENAME]["etag"] == "6423adf134a669acc357f619d1162009" + assert fp["headers"][_FILENAME]["content_length"] == "10270511" + + +def test_content_signal_is_not_recorded_as_a_schema_checksum(): + """`drift_to_pr` reads `checksums` as a hash of the column-header row, so a raw + ETag there tiers every routine content update as a SCHEMA change and tells the + reviewer to check builder.py. This probe fetches no body and has no schema + signal to offer, so `checksums` must stay empty.""" + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers=_RESPONSE_HEADERS) + fp = fetch_fingerprint() + + assert fp["checksums"] == {} + assert fp["headers"], "signal must still be compared, just not as a schema hash" + + +def test_last_modified_is_demoted_out_of_the_compared_surface(): + """A byte-identical republish must not open a pull request (hgnc precedent).""" + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers=_RESPONSE_HEADERS) + fp = fetch_fingerprint() + + assert fp["source_version"] is None + assert fp["informational"]["last_modified"] == "Tue, 14 Nov 2023 18:23:50 GMT" + # The negative half: the timestamp must appear nowhere that drift compares. + assert "last_modified" not in fp["headers"][_FILENAME] + assert "last_modified" not in fp["checksums"] + + +def test_weak_etag_normalizes_to_the_same_tag_as_the_strong_form(): + """CDNs flip strong<->weak on a byte-identical object. `strip('"')` left the + `W/` prefix attached, so the mangled value read as drift and was then committed + as the new baseline.""" + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers=_RESPONSE_HEADERS) + strong = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.head( + GEVIR_SUPPLEMENTARY_URL, + headers={ + **_RESPONSE_HEADERS, + "ETag": 'W/"6423adf134a669acc357f619d1162009"', + }, + ) + weak = fetch_fingerprint() + + assert weak["headers"] == strong["headers"] + + +def test_no_content_signal_fails_closed(): + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers={}) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_fingerprint() + + +def test_etag_only_response_fails_closed(): + """Accepting it would record `content_length: None`; the bot commits that as the + new baseline and every later normal response then reads as drift forever.""" + with requests_mock.Mocker() as m: + m.head( + GEVIR_SUPPLEMENTARY_URL, + headers={"ETag": '"6423adf134a669acc357f619d1162009"'}, + ) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_fingerprint() + + +def test_empty_etag_fails_closed(): + with requests_mock.Mocker() as m: + m.head( + GEVIR_SUPPLEMENTARY_URL, + headers={"Content-Length": "10270511", "ETag": '""'}, + ) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_fingerprint() + + +def test_probe_requests_identity_encoding(): + """Without this a compressing server drops Content-Length and suffixes the ETag.""" + with requests_mock.Mocker() as m: + m.head(GEVIR_SUPPLEMENTARY_URL, headers=_RESPONSE_HEADERS) + fetch_fingerprint() + assert len(m.request_history) == 1 + assert m.request_history[0].headers.get("Accept-Encoding") == "identity" + + +def test_compressed_response_fails_closed(): + with requests_mock.Mocker() as m: + m.head( + GEVIR_SUPPLEMENTARY_URL, + headers={ + "Content-Length": "123", + "ETag": '"abc"', + "Content-Encoding": "gzip", + }, + ) + with pytest.raises(DriftProbeError, match="gzip-encoded"): + fetch_fingerprint() diff --git a/hvantk/skills/gnomad_metrics/SKILL.md b/hvantk/skills/gnomad_metrics/SKILL.md index 8f3b6afb..7b9978b8 100644 --- a/hvantk/skills/gnomad_metrics/SKILL.md +++ b/hvantk/skills/gnomad_metrics/SKILL.md @@ -59,8 +59,11 @@ from `plugin.yaml`; top-level builds run through `run_builder_for_spec` ## Phase K notes This plugin was promoted as part of Phase K of the data-model platform -refactor. The drift probe (`drift_probe.py`, `fetch_fingerprint`) is a -stub; a real probe should be implemented in a follow-up. The downloader +refactor. The drift probe (`drift_probe.py`, `fetch_fingerprint`) HEADs every +declared constraint object in the public `gcp-public-data--gnomad` bucket and compares +the MD5 ETag, Content-Length and `x-goog-generation` per object; no body is +transferred. It is the strongest comparator in the tree, since GCS ETags are content +digests rather than mtime-derived validators. The downloader (`cli.py`, `download_dataset` / `download_cmd`) fetches the constraint tables from the public gnomAD GCS bucket — see the Download section above. diff --git a/hvantk/skills/gnomad_metrics/drift_probe.py b/hvantk/skills/gnomad_metrics/drift_probe.py index 254537e2..b36b8d79 100644 --- a/hvantk/skills/gnomad_metrics/drift_probe.py +++ b/hvantk/skills/gnomad_metrics/drift_probe.py @@ -1,21 +1,157 @@ -"""Drift probe for gnomad-metrics — documentation-only source (stub). +"""gnomad-metrics drift probe: HEAD against the public constraint tables. -gnomAD constraint metrics are acquired manually here and ship no committed -baseline fingerprint. A HEAD probe of the public v2.1.1 constraint file on GCS is -marginally feasible but deferred (see issue #177, Option 2). For now this probe -returns a structured stub sentinel so ``hvantk drift`` reports a visible WARNING -(status="stub") rather than a silent false-green. +The constraint tables live in the ``gcp-public-data--gnomad`` release bucket over +plain HTTPS with no auth, so every table the builder can consume is directly +addressable. Issue #177 rated this source "marginally feasible (fragile, +network-only)"; in practice GCS returns a stronger fingerprint than the hgnc +reference does, because for a simple (non-composite) object the ETag is an MD5 +over the body rather than an mtime-derived validator. + +All declared tables are probed rather than only the default, so the fingerprint +covers whichever release a given build pinned. Each is a HEAD, so nothing is +transferred (the v4.0 table alone is 86 MB), and all of them share one +``requests.Session`` so the three objects cost one TLS handshake. + +Compared surface: the MD5 ETag and Content-Length per object, plus +``x-goog-generation`` -- a GCS counter that changes on every object rewrite even +if the bytes happen to be identical, which makes a silent republish visible. +These live under ``headers`` rather than ``checksums`` because ``_conventions`` +§ 12 defines ``checksums`` as "sha256 over the bytes used to derive ``headers``"; +this probe fetches no body and computes no such digest, so recording a raw +validator there would misrepresent the contract. ``Last-Modified`` is demoted to +``informational`` following the hgnc precedent, where 8 of 8 drift PRs moved only +the timestamp. + +Note that ``headers`` is a drift-bot schema key, so any change here is tiered +"schema" rather than "routine". That is deliberate and follows +``classify_risk``'s own stated policy -- "Defaults to schema for anything it +cannot read. Misclassifying a schema change as routine would bury it in a batch; +the reverse just opens one extra PR." A HEAD-only probe cannot see column names, +so the conservative tier is the correct one. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +from datetime import datetime, timezone + +import requests -_REASON = ( - "gnomAD constraint metrics acquired manually; no committed baseline " - "(a GCS HEAD probe of the public v2.1.1 file is feasible but deferred — " - "issue #177)" +from hvantk.core.plugin.api import DriftProbeError, normalize_etag +from hvantk.core.utils.http import request_with_retry +from hvantk.skills.gnomad_metrics.shared.constants import ( + GNOMAD_CONSTRAINT_TABLES, + GNOMAD_RELEASE_BASE_URL, ) +PROBE_VERSION = 2 + +# Evaluated once: GNOMAD_CONSTRAINT_TABLES is a module constant and cannot change +# at runtime. +OBJECT_PATHS: tuple[str, ...] = tuple( + sorted( + path + for tables in GNOMAD_CONSTRAINT_TABLES.values() + for path in tables.values() + ) +) + +# Sized against the runner's budget, not per request. drift_cli defaults +# --timeout to 60s and enforces it with a single SIGALRM around the whole probe, +# while requests applies its timeout separately to connect and read. At the +# previous 30s this loop's worst case was 3 x 60 = 180s, so a merely slow bucket +# reported probe_failed on a healthy source. A (connect, read) pair keeps the +# whole loop inside the alarm. +_TIMEOUT_S = (5.0, 10.0) + +# A server that compresses on the fly omits Content-Length and mangles the ETag. +_HEADERS = {"Accept-Encoding": "identity"} + + +def _head_object( + session: requests.Session, path: str +) -> tuple[dict, str | None]: + """Return (compared signals, Last-Modified) for one object, or raise.""" + url = f"{GNOMAD_RELEASE_BASE_URL}/{path}" + try: + head = request_with_retry( + "HEAD", + url, + session=session, + timeout=_TIMEOUT_S, + allow_redirects=True, + headers=_HEADERS, + ) + head.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure for {path}: {exc}") from exc + + content_length = head.headers.get("Content-Length") + etag = normalize_etag(head.headers.get("ETag")) + + # Fail closed on emptiness, and require BOTH validators. Accepting one alone + # was the hole: an ETag-only response recorded a null length, and a + # Content-Length-only response silently dropped the object's key out of + # `headers` entirely -- after which the bot regenerates, commits the degraded + # baseline, and the MD5 signal for that table is gone for good while drift + # keeps reporting clean. + if not content_length or not etag: + raise DriftProbeError( + f"gnomAD response for {path} carried no usable content signal " + f"(Content-Length={content_length!r}, ETag={etag!r}); refusing to " + "record a fingerprint that would compare equal to its baseline " + "forever." + ) + + # Asserted request-side above; verified response-side here, because a caching + # proxy may compress regardless and the recorded length would then describe + # the compressed body rather than the object. + encoding = (head.headers.get("Content-Encoding") or "identity").lower() + if encoding != "identity": + raise DriftProbeError( + f"gnomAD response for {path} was {encoding}-encoded despite an " + "identity request; Content-Length would describe the compressed body." + ) + + compared = { + "content_length": content_length, + "etag": etag, + "generation": head.headers.get("x-goog-generation"), + } + return compared, head.headers.get("Last-Modified") + def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Lightweight fingerprint of the live gnomAD constraint tables (HEAD only).""" + headers: dict[str, dict[str, str | None]] = {} + informational: dict[str, str | None] = {} + failures: list[str] = [] + + with requests.Session() as session: + for path in OBJECT_PATHS: + try: + compared, last_modified = _head_object(session, path) + except DriftProbeError as exc: + # Collected rather than raised immediately: raising on the first + # failure meant a fault on either frozen-since-2020 v2.1.1 object + # aborted before v4.0 -- the current release -- was ever probed, + # so a genuine v4.0 republish stayed invisible for as long as the + # unrelated fault persisted. + failures.append(str(exc)) + else: + headers[path] = compared + informational[path] = last_modified + + if failures: + raise DriftProbeError( + f"{len(failures)} of {len(OBJECT_PATHS)} gnomAD objects could not be " + "fingerprinted: " + "; ".join(failures) + ) + + return { + "probe_version": PROBE_VERSION, + "source_version": None, + "headers": headers, + "checksums": {}, + "informational": informational, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/gnomad_metrics/tests/drift_fingerprint.json b/hvantk/skills/gnomad_metrics/tests/drift_fingerprint.json new file mode 100644 index 00000000..d6c53044 --- /dev/null +++ b/hvantk/skills/gnomad_metrics/tests/drift_fingerprint.json @@ -0,0 +1,28 @@ +{ + "probe_version": 2, + "source_version": null, + "headers": { + "2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz": { + "content_length": "4609488", + "etag": "967fb59dc509d9ee351b9d35efacf52a", + "generation": "1597936745580178" + }, + "2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_transcript.txt.bgz": { + "content_length": "13341280", + "etag": "ee84bf0398e551b2ad52608cefb3ce30", + "generation": "1597936751635121" + }, + "v4.0/constraint/gnomad.v4.0.constraint_metrics.tsv": { + "content_length": "85888818", + "etag": "612f3899a85a1b7c6513a9dd65b00485", + "generation": "1699056035127872" + } + }, + "checksums": {}, + "informational": { + "2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz": "Thu, 20 Aug 2020 15:19:05 GMT", + "2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_transcript.txt.bgz": "Thu, 20 Aug 2020 15:19:11 GMT", + "v4.0/constraint/gnomad.v4.0.constraint_metrics.tsv": "Sat, 04 Nov 2023 00:00:35 GMT" + }, + "fetched_at": "2026-09-01T07:32:17.657594+00:00" +} diff --git a/hvantk/skills/gnomad_metrics/tests/test_drift_probe.py b/hvantk/skills/gnomad_metrics/tests/test_drift_probe.py new file mode 100644 index 00000000..e3b8a48e --- /dev/null +++ b/hvantk/skills/gnomad_metrics/tests/test_drift_probe.py @@ -0,0 +1,137 @@ +"""gnomad-metrics drift probe should fingerprint every constraint object offline. + +Runs OFFLINE via requests_mock. The probe replaced a stub sentinel (issue #177, +which rated the source "marginally feasible") once the public GCS objects were +confirmed to answer a HEAD with an MD5 ETag. +""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.gnomad_metrics.drift_probe import OBJECT_PATHS, fetch_fingerprint +from hvantk.skills.gnomad_metrics.shared.constants import GNOMAD_RELEASE_BASE_URL + +_BY_GENE = "2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_gene.txt.bgz" + + +def _url(path): + return f"{GNOMAD_RELEASE_BASE_URL}/{path}" + + +def _headers(n): + """Distinct per object, so a probe that mixed them up would be caught.""" + return { + "ETag": f'"{n:032x}"', + "Content-Length": str(1000 + n), + "x-goog-generation": str(1_500_000_000_000_000 + n), + "Last-Modified": f"Thu, 20 Aug 2020 15:19:{n:02d} GMT", + } + + +def _mock_all(m, overrides=None): + for n, path in enumerate(OBJECT_PATHS): + m.head(_url(path), headers=(overrides or {}).get(path, _headers(n))) + + +def test_each_object_lands_under_its_own_key(): + """Distinct fixtures per object, so assigning one object's validators to all + three would fail here.""" + with requests_mock.Mocker() as m: + _mock_all(m) + fp = fetch_fingerprint() + + assert set(fp["headers"]) == set(OBJECT_PATHS) + for n, path in enumerate(OBJECT_PATHS): + assert fp["headers"][path]["content_length"] == str(1000 + n) + assert fp["headers"][path]["etag"] == f"{n:032x}" + + +def test_signals_are_compared_not_stashed_in_checksums(): + """`_conventions` §12 defines `checksums` as a sha256 over the bytes used to + derive `headers`. This probe fetches no body, so recording a raw validator + there would misrepresent the contract.""" + with requests_mock.Mocker() as m: + _mock_all(m) + fp = fetch_fingerprint() + + assert fp["checksums"] == {} + assert fp["headers"], "signal must still be compared" + + +def test_generation_change_is_visible(): + """x-goog-generation moves on every rewrite even when bytes are identical.""" + with requests_mock.Mocker() as m: + _mock_all(m) + before = fetch_fingerprint() + + bumped = {OBJECT_PATHS[0]: {**_headers(0), "x-goog-generation": "9999"}} + with requests_mock.Mocker() as m: + _mock_all(m, overrides=bumped) + after = fetch_fingerprint() + + assert before["headers"] != after["headers"] + + +def test_last_modified_is_demoted_out_of_the_compared_surface(): + with requests_mock.Mocker() as m: + _mock_all(m) + fp = fetch_fingerprint() + + assert fp["source_version"] is None + assert fp["informational"][_BY_GENE].startswith("Thu, 20 Aug 2020") + assert "last_modified" not in fp["headers"][_BY_GENE] + + +@pytest.mark.parametrize( + ("label", "bad_headers"), + [ + ("no headers at all", {}), + ("content-length only", {"Content-Length": "4609488"}), + ("etag only", {"ETag": '"abc"'}), + ("empty etag", {"Content-Length": "4609488", "ETag": '""'}), + ], +) +def test_partial_validators_fail_closed(label, bad_headers): + """Each discriminating case, not just the both-missing one. An ETag-only + response used to record a null length; a Content-Length-only response used to + drop the object out of the compared surface entirely.""" + with requests_mock.Mocker() as m: + _mock_all(m, overrides={OBJECT_PATHS[0]: bad_headers}) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_fingerprint() + + +def test_compressed_response_fails_closed(): + bad = {**_headers(0), "Content-Encoding": "gzip"} + with requests_mock.Mocker() as m: + _mock_all(m, overrides={OBJECT_PATHS[0]: bad}) + with pytest.raises(DriftProbeError, match="gzip-encoded"): + fetch_fingerprint() + + +def test_one_object_failing_still_reports_the_others(): + """Raising on the first failure meant a fault on either frozen v2.1.1 object + aborted before v4.0 -- the current release -- was ever probed. The error must + name what failed rather than stopping at the first.""" + with requests_mock.Mocker() as m: + _mock_all(m) + m.head(_url(OBJECT_PATHS[0]), status_code=503) + with pytest.raises(DriftProbeError) as excinfo: + fetch_fingerprint() + + msg = str(excinfo.value) + assert f"1 of {len(OBJECT_PATHS)}" in msg + assert OBJECT_PATHS[0] in msg + + +def test_probe_requests_identity_encoding(): + with requests_mock.Mocker() as m: + _mock_all(m) + fetch_fingerprint() + assert len(m.request_history) == len(OBJECT_PATHS) + assert all( + r.headers.get("Accept-Encoding") == "identity" for r in m.request_history + ) diff --git a/hvantk/skills/insider/SKILL.md b/hvantk/skills/insider/SKILL.md index 535d8a50..54cadd2f 100644 --- a/hvantk/skills/insider/SKILL.md +++ b/hvantk/skills/insider/SKILL.md @@ -24,7 +24,10 @@ This skill is the **first interval-keyed skill** in hvantk. Conventions § 3 dec ## 2. Source identity - **Provider:** Yu lab (Cornell). Wei et al., *Nat Methods* 2017, PMID 29036289. -- **Distribution:** http://interactomeinsider.yulab.org/downloads.html +- **Distribution:** the download page carries no links in its markup, but both + products have stable, directly addressable paths. Per `_conventions` § 2 the + catalog owns them: see `catalog/datasets.json`, or run + `hvantk catalog show INSIDER_v1.0`. Both are what the drift probes pin. - **License:** Academic use (per the existing catalog entry). - **Catalog entry:** `INSIDER_v1.0` in `hvantk/resources/registry/genomics/datasets.json`. **Filename and metadata corrected in the same PR that adds this skill** — the prior entry listed `insider_interaction_sites.tsv` which is not a real INSIDER distribution product (see § 4 Gap 2). @@ -115,6 +118,38 @@ Per `_conventions` § 9: - **schema_snapshot:** `hvantk/skills/insider/tests/snapshots/schema.json`. Records the `{interval, ppi_ids: array}` shape. - **row_snapshot:** `hvantk/skills/insider/tests/snapshots/sample_rows.json`. Intervals are unique-in-table after the aggregation; test inlines 3 sample keys (per `_conventions` § 9 post-#101 rule — unique-key skills inline). - **test_command:** `pytest hvantk/skills/insider/tests -m hail`. +- **drift_fingerprint:** `hvantk/skills/insider/tests/drift_fingerprint.json` for + `variants`, and `interfaces/tests/drift_fingerprint.json` for `interfaces` — **one + baseline per dataset, deliberately not shared.** The two products are versioned + independently upstream (the BED has not moved since 2018-03-05; the interfaces table + moved 2024-05-15), and a shared probe/baseline had three consequences: the drift bot + writes a ledger row only for a group's anchor dataset, so an interfaces-only change + could never be reported against the dataset that needed rebuilding; a transient fault + on the BED aborted the probe before the interfaces file was reached, masking real + drift; and each artifact's provenance covered the other dataset's file. Fetch + mechanics are shared in `shared/http_probe.py`; the fingerprints are not. + + Each probe HEADs its own file (§ 2) and compares **Content-Length and the ETag**, + both under `headers`; no body is transferred. Two details are load-bearing: + + - The ETag is *not* redundant with Content-Length. The live values decode as + `hex(size)-hex(mtime)` — `0x2f752a5` is exactly the interfaces file's 49,762,981 + bytes — so it moves on size *or* mtime, while Content-Length moves only on size. + Demoting it would make every equal-size edit (a swapped accession, a corrected + residue index) undetectable. The hgnc precedent for demoting validators does not + transfer: hgnc republishes byte-identical content weekly, whereas these are static + archives, so the no-op-PR risk is near-nil. `Last-Modified` stays in + `informational`. + - The signals sit under `headers`, not `extras`, because `drift_to_pr.classify_risk` + treats a diff as "routine" unless `headers` or `checksums` moved. With both empty, + a re-release that changed the `track name=` format — the case § 8 says breaks the + parser — would be batched under a PR body stating the schema signal was unchanged. + + The probes request `Accept-Encoding: identity` and reject a non-identity response: + the server gzips `text/plain` on the fly, and a compressed reply omits + `Content-Length` entirely while appending `-gzip` to the ETag. Note the portal + serves **no HTTPS listener**, so these are cleartext requests; comparing two + independent validators rather than one is a mitigation, not a fix. Round-trip test (`hvantk/skills/insider/tests/test_builder.py`, via `phase_b_snapshot_adapter(build_insider_interactome, "insider:variants")`) asserts: checkpointed schema matches `schema.json`; deterministic sample-row slice matches `sample_rows.json`. The test exercises the `hl.tinterval` handling in `_snapshot_utils` (added in PR #105) — if that branch breaks, this test breaks. diff --git a/hvantk/skills/insider/catalog/datasets.json b/hvantk/skills/insider/catalog/datasets.json index 5617f7a9..0325a4e1 100644 --- a/hvantk/skills/insider/catalog/datasets.json +++ b/hvantk/skills/insider/catalog/datasets.json @@ -5,7 +5,7 @@ "description": "Interactome Insider (Wei et al., Nat Methods 2017). Predicted (ECLAIR), structural (PDB), and homology-derived (I3D) protein-protein interaction interface residues. Distributed as two complementary products: (1) Whole_Human_Interactome_Interface_hg38.bed \u2014 UCSC-style BED with 208,448 named PPI tracks projecting interface residues onto GRCh38 coordinates (this file path), and (2) H_sapiens_interfacesALL.txt \u2014 per protein-pair residue arrays with Source provenance (a separately-onboardable product, not in this catalog entry). Built by a custom track-aware BED parser plus hl.import_table into an interval-keyed Hail Table that preserves PPI IDs from track names; see hvantk/skills/insider/SKILL.md. Used for variant-interval intersection (does a variant fall in any predicted PPI interface?).", "pubmedid": "29036289", "data_source": "INSIDER", - "last_updated": "2025-09-23T16:30:00.000000", + "last_updated": "2026-09-01T00:00:00.000000", "update_frequency": "irregular", "organism": "Homo sapiens", "tissue_type": "all", @@ -27,9 +27,16 @@ { "path": "Whole_Human_Interactome_Interface_hg38.bed", "format": "bed", - "size_bytes": 1171586071, - "url": "http://interactomeinsider.yulab.org/downloads.html", + "size_bytes": 1171848841, + "url": "http://interactomeinsider.yulab.org/bed/all.bed", "description": "UCSC-style BED with 208,448 'track name=_ppi_' headers; 18.6M data rows representing per-residue genomic intervals (typically 1-3 bp) for predicted/experimental protein-protein interface positions. >1 GB; manual acquisition per CLAUDE.md downloader strategy. The track-aware INSIDER builder preserves track-name metadata as PPI IDs during import." + }, + { + "path": "H_sapiens_interfacesALL.txt", + "format": "tsv", + "size_bytes": 49762981, + "url": "http://interactomeinsider.yulab.org/downloads/interfacesALL/H_sapiens_interfacesALL.txt", + "description": "Per-protein-pair interface residue arrays with Source provenance (ECLAIR predicted / PDB / I3D homology). Backs `insider:interfaces`; protein-keyed rather than genomic, so no coordinate join is needed. Direct path: the downloads page carries no links in its markup." } ] } diff --git a/hvantk/skills/insider/drift_probe.py b/hvantk/skills/insider/drift_probe.py index 97921d18..7982e79c 100644 --- a/hvantk/skills/insider/drift_probe.py +++ b/hvantk/skills/insider/drift_probe.py @@ -1,26 +1,37 @@ -"""INSIDER drift probe — documentation-only source (stub). - -The Interactome Insider portal (http://interactomeinsider.yulab.org) hosts -the ``Whole_Human_Interactome_Interface_hg38.bed`` file behind a manual -download page with no machine-readable manifest or release feed; a real -drift probe would need to track the page contents or the file's -Last-Modified/ETag header. Writing that probe is out of scope for the -Phase 1 migration. - -Until a real probe lands, ``fetch_fingerprint`` returns a structured stub -sentinel (via :func:`hvantk.core.plugin.api.stub_fingerprint`) so -``hvantk drift insider:variants`` reports a visible WARNING (status="stub") -instead of a silent false-green "clean". See issue #177. +"""INSIDER drift probe for ``insider:variants``: HEAD against the genomic BED. + +Issue #177 recorded the source as needing a page-scraping probe, because the +download page carries no links in its markup. The underlying path is stable and +directly addressable, so no scraping is required. + +This probe covers ``/bed/all.bed`` only -- the >1 GB +``Whole_Human_Interactome_Interface_hg38.bed`` the builder reads. Confirmed to be +that product rather than a same-named sibling: its first bytes are the +``browser hide all`` directive followed by ``track name=A0A0A0MS80_ppi_P56705`` +and ``chr11 700235``, matching the committed fixture and the excerpt in +SKILL.md § 4. + +``insider:interfaces`` has its own probe and its own baseline +(``interfaces/drift_probe.py``); see ``shared/http_probe.py`` for why the two +must not share one. + +Only a HEAD is issued, so the 1.17 GB body is never transferred. """ from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint - -NOT_IMPLEMENTED_REASON = ( - "insider drift probe not implemented; track upstream release manually" +from hvantk.skills.insider.shared.http_probe import ( + INSIDER_BASE_URL, + head_fingerprint, ) +INSIDER_BED_URL = f"{INSIDER_BASE_URL}/bed/all.bed" + +# The documented product name, not the URL basename (`all.bed`), so the +# fingerprint reads against the name SKILL.md and the catalog use. +INSIDER_BED_FILENAME = "Whole_Human_Interactome_Interface_hg38.bed" + def fetch_fingerprint() -> dict: - return stub_fingerprint(NOT_IMPLEMENTED_REASON) + """Fingerprint the live genomic BED (HEAD only).""" + return head_fingerprint(INSIDER_BED_FILENAME, INSIDER_BED_URL) diff --git a/hvantk/skills/insider/interfaces/drift_probe.py b/hvantk/skills/insider/interfaces/drift_probe.py new file mode 100644 index 00000000..7b93eaa6 --- /dev/null +++ b/hvantk/skills/insider/interfaces/drift_probe.py @@ -0,0 +1,34 @@ +"""INSIDER drift probe for ``insider:interfaces``: HEAD against the pair table. + +Covers ``/downloads/interfacesALL/H_sapiens_interfacesALL.txt`` (~49 MB) only -- +the protein-pair product this dataset builds from, which is a different file from +the genomic BED behind ``insider:variants``. + +Deliberately separate from the sibling probe, with its own committed baseline. +The two products are versioned independently upstream (the BED has not moved +since 2018-03-05; this table moved 2024-05-15), and sharing one probe and one +baseline had three consequences: the drift bot writes a ledger row only for the +group's anchor dataset, so an interfaces-only change could never be reported +against the dataset that actually needed rebuilding; a transient fault on the BED +aborted the probe before this file was ever reached, masking real drift here; and +each artifact's recorded provenance covered the other dataset's file as well as +its own. +""" + +from __future__ import annotations + +from hvantk.skills.insider.shared.http_probe import ( + INSIDER_BASE_URL, + head_fingerprint, +) + +INSIDER_INTERFACES_URL = ( + f"{INSIDER_BASE_URL}/downloads/interfacesALL/H_sapiens_interfacesALL.txt" +) + +INSIDER_INTERFACES_FILENAME = "H_sapiens_interfacesALL.txt" + + +def fetch_fingerprint() -> dict: + """Fingerprint the live protein-pair interfaces table (HEAD only).""" + return head_fingerprint(INSIDER_INTERFACES_FILENAME, INSIDER_INTERFACES_URL) diff --git a/hvantk/skills/insider/interfaces/tests/drift_fingerprint.json b/hvantk/skills/insider/interfaces/tests/drift_fingerprint.json new file mode 100644 index 00000000..7ffeb07e --- /dev/null +++ b/hvantk/skills/insider/interfaces/tests/drift_fingerprint.json @@ -0,0 +1,17 @@ +{ + "probe_version": 2, + "source_version": null, + "headers": { + "H_sapiens_interfacesALL.txt": { + "content_length": "49762981", + "etag": "2f752a5-61883677f7fa2" + } + }, + "checksums": {}, + "informational": { + "H_sapiens_interfacesALL.txt": { + "last_modified": "Wed, 15 May 2024 19:48:36 GMT" + } + }, + "fetched_at": "2026-09-01T00:41:41.783451+00:00" +} diff --git a/hvantk/skills/insider/plugin.yaml b/hvantk/skills/insider/plugin.yaml index 8ad27ddf..180dfa6c 100644 --- a/hvantk/skills/insider/plugin.yaml +++ b/hvantk/skills/insider/plugin.yaml @@ -43,7 +43,7 @@ datasets: module: hvantk.skills.insider.interfaces.builder function: build_insider_interfaces drift_probe: - module: hvantk.skills.insider.drift_probe + module: hvantk.skills.insider.interfaces.drift_probe function: fetch_fingerprint skill: SKILL.md tests: @@ -51,4 +51,4 @@ datasets: fixture: interfaces/tests/testdata/raw/interfaces schema_snapshot: interfaces/tests/snapshots/schema.json row_snapshot: interfaces/tests/snapshots/sample_rows.json - drift_fingerprint: tests/drift_fingerprint.json + drift_fingerprint: interfaces/tests/drift_fingerprint.json diff --git a/hvantk/skills/insider/shared/http_probe.py b/hvantk/skills/insider/shared/http_probe.py new file mode 100644 index 00000000..646742a7 --- /dev/null +++ b/hvantk/skills/insider/shared/http_probe.py @@ -0,0 +1,106 @@ +"""Shared HEAD-fingerprint helper for the two INSIDER products. + +``insider:variants`` and ``insider:interfaces`` read two different files from one +portal, so they share the fetch mechanics but must NOT share a fingerprint: they +are versioned independently upstream (the BED has not moved since 2018-03-05 +while the interfaces table moved 2024-05-15), and a shared baseline makes the +drift bot record a ledger row only for the group's anchor dataset, so an +interfaces-only change can never be reported against the dataset that actually +needs rebuilding. Each dataset therefore calls this with its own file and commits +its own baseline. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError, normalize_etag + +PROBE_VERSION = 2 + +INSIDER_BASE_URL = "http://interactomeinsider.yulab.org" + +# The portal serves no HTTPS listener (a TLS connection to the same host fails), +# so the probe is forced onto cleartext. That is a real exposure: an interposing +# proxy can define what this records. It is mitigated, not solved, by comparing +# two independent validators rather than one -- an interstitial would have to +# forge a consistent (Content-Length, ETag) pair to pass unnoticed -- and by the +# fact that any such forgery reads as drift rather than as clean. +_TIMEOUT_S = 30 + +# The portal compresses text/plain on the fly when the client offers it, and a +# compressed response omits Content-Length entirely while appending "-gzip" to +# the ETag. requests sends "gzip, deflate" by default, so the probe must opt out +# or the interfaces file's signals disappear. +_HEADERS = {"Accept-Encoding": "identity"} + + +def head_fingerprint(filename: str, url: str) -> dict: + """Fingerprint one INSIDER product by HEAD, transferring no body. + + Compared surface: Content-Length **and** the ETag, both under ``headers``. + + Two corrections over the first version of this probe. The ETag is not + redundant with Content-Length: decoding the live values shows the tag is + ``hex(size)-hex(mtime)`` (``0x2f752a5`` is exactly the interfaces file's + 49,762,981-byte length), so it moves on size *or* mtime while Content-Length + moves only on size. Demoting it therefore made every equal-size edit -- a + swapped accession, a corrected residue index -- undetectable. The hgnc + precedent for demoting validators does not transfer: hgnc republishes + byte-identical content weekly, whereas these two objects are static archives + that have not moved in years, so the no-op-PR risk is near-nil and the signal + given up was the only one that catches an equal-size change. + + And the signals live under ``headers`` rather than ``extras`` because the + drift bot reads ``headers``/``checksums`` as its schema signal and tiers + anything else as "routine". With both of those empty, a re-release that + changed the ``track name=`` format -- the case insider/SKILL.md § 8 says + breaks the parser -- would have been swept into a batch whose body tells the + reviewer the schema signal is unchanged. peptideatlas ships this same shape: + validator metadata under ``headers``, ``checksums`` left empty. + """ + try: + head = requests.head( + url, timeout=_TIMEOUT_S, allow_redirects=True, headers=_HEADERS + ) + head.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure for {filename}: {exc}") from exc + + content_length = head.headers.get("Content-Length") + etag = normalize_etag(head.headers.get("ETag")) + + # Fail closed on emptiness, not just absence: a proxy answering `ETag: ""` + # would otherwise record an empty digest, which placeholder_baseline_reason + # later reads as a hand-seeded baseline -- trapping the dataset in a + # probe_failed loop that regenerating cannot clear. + if not content_length or not etag: + raise DriftProbeError( + f"INSIDER response for {filename} carried no usable content signal " + f"(Content-Length={content_length!r}, ETag={etag!r}); refusing to " + "record a fingerprint that would compare equal to its baseline " + "forever." + ) + + # Asserted request-side above; verified response-side here, because a + # caching proxy or gzip_static may compress regardless and the recorded + # length would then describe the compressed body rather than the object. + encoding = (head.headers.get("Content-Encoding") or "identity").lower() + if encoding != "identity": + raise DriftProbeError( + f"INSIDER response for {filename} was {encoding}-encoded despite an " + "identity request; Content-Length would describe the compressed body." + ) + + return { + "probe_version": PROBE_VERSION, + "source_version": None, + "headers": {filename: {"content_length": content_length, "etag": etag}}, + "checksums": {}, + "informational": { + filename: {"last_modified": head.headers.get("Last-Modified")} + }, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/insider/tests/drift_fingerprint.json b/hvantk/skills/insider/tests/drift_fingerprint.json index d8438207..1fe2daa4 100644 --- a/hvantk/skills/insider/tests/drift_fingerprint.json +++ b/hvantk/skills/insider/tests/drift_fingerprint.json @@ -1,5 +1,17 @@ { - "probe_status": "stub", - "reason": "insider drift probe not implemented; track upstream release manually", - "fingerprint": "stub:no-programmatic-source" + "probe_version": 2, + "source_version": null, + "headers": { + "Whole_Human_Interactome_Interface_hg38.bed": { + "content_length": "1171848841", + "etag": "45d8fe89-566adb9ac3780" + } + }, + "checksums": {}, + "informational": { + "Whole_Human_Interactome_Interface_hg38.bed": { + "last_modified": "Mon, 05 Mar 2018 17:33:34 GMT" + } + }, + "fetched_at": "2026-09-01T00:41:41.500835+00:00" } diff --git a/hvantk/skills/insider/tests/test_drift_probe.py b/hvantk/skills/insider/tests/test_drift_probe.py index 27376c72..83da9b20 100644 --- a/hvantk/skills/insider/tests/test_drift_probe.py +++ b/hvantk/skills/insider/tests/test_drift_probe.py @@ -1,21 +1,173 @@ -"""Sanity test for the INSIDER stub drift probe. +"""INSIDER drift probes should fingerprint each product independently. -The INSIDER source has no programmatically-probeable URL, so the probe returns -a structured stub sentinel (status="stub" + reason) instead of a false-green -constant. This asserts that shape so a regression back to the silent -false-green cannot slip through. See issue #177. +Runs OFFLINE via requests_mock, so CI never hits the live portal. The probes +replaced a stub sentinel (issue #177) once the portal's direct file paths were +confirmed addressable; these tests pin the properties that make them real +comparators rather than false-greens. """ -from hvantk.core.plugin.api import PROBE_STATUS_STUB, STUB_FINGERPRINT_TOKEN +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError from hvantk.skills.insider.drift_probe import ( - NOT_IMPLEMENTED_REASON, + INSIDER_BED_FILENAME, + INSIDER_BED_URL, fetch_fingerprint, ) +from hvantk.skills.insider.interfaces.drift_probe import ( + INSIDER_INTERFACES_FILENAME, + INSIDER_INTERFACES_URL, +) +from hvantk.skills.insider.interfaces.drift_probe import ( + fetch_fingerprint as fetch_interfaces_fingerprint, +) + +_BED_HEADERS = { + "Content-Length": "1171848841", + "ETag": '"45d8fe89-566adb9ac3780"', + "Last-Modified": "Mon, 05 Mar 2018 17:33:34 GMT", +} +_INTERFACES_HEADERS = { + "Content-Length": "49762981", + "ETag": '"2f752a5-61883677f7fa2"', + "Last-Modified": "Wed, 15 May 2024 19:48:36 GMT", +} + + +def test_bed_probe_fingerprints_only_its_own_file(): + """The two products are versioned independently, so a variants fingerprint + must not carry the interfaces file -- otherwise an interfaces-only change + rewrites the provenance of every BED-derived artifact.""" + with requests_mock.Mocker() as m: + m.head(INSIDER_BED_URL, headers=_BED_HEADERS) + fp = fetch_fingerprint() + + assert set(fp["headers"]) == {INSIDER_BED_FILENAME} + assert INSIDER_INTERFACES_FILENAME not in fp["headers"] + assert fp["headers"][INSIDER_BED_FILENAME]["content_length"] == "1171848841" + + +def test_interfaces_probe_fingerprints_only_its_own_file(): + with requests_mock.Mocker() as m: + m.head(INSIDER_INTERFACES_URL, headers=_INTERFACES_HEADERS) + fp = fetch_interfaces_fingerprint() + + assert set(fp["headers"]) == {INSIDER_INTERFACES_FILENAME} + assert INSIDER_BED_FILENAME not in fp["headers"] + + +def test_a_bed_outage_cannot_mask_interfaces_drift(): + """Sharing one probe meant a fault on the frozen-since-2018 BED aborted before + the interfaces file was ever reached. Split probes must be independent.""" + with requests_mock.Mocker() as m: + m.head(INSIDER_BED_URL, status_code=503) + m.head(INSIDER_INTERFACES_URL, headers=_INTERFACES_HEADERS) + + with pytest.raises(DriftProbeError): + fetch_fingerprint() + # The interfaces probe is unaffected. + fp = fetch_interfaces_fingerprint() + + assert fp["headers"][INSIDER_INTERFACES_FILENAME]["content_length"] == "49762981" + + +def test_etag_is_in_the_compared_surface(): + """The ETag is hex(size)-hex(mtime), a strict superset of Content-Length, so it + catches an equal-size edit that Content-Length alone cannot. It must sit under + `headers` (compared), not `informational` (ignored by drift comparison).""" + with requests_mock.Mocker() as m: + m.head(INSIDER_INTERFACES_URL, headers=_INTERFACES_HEADERS) + fp = fetch_interfaces_fingerprint() + assert fp["headers"][INSIDER_INTERFACES_FILENAME]["etag"] == "2f752a5-61883677f7fa2" + assert "etag" not in fp.get("informational", {}).get( + INSIDER_INTERFACES_FILENAME, {} + ) -def test_stub_fingerprint_shape(): - fp = fetch_fingerprint() - assert fp["probe_status"] == PROBE_STATUS_STUB - assert fp["fingerprint"] == STUB_FINGERPRINT_TOKEN - assert fp["reason"] == NOT_IMPLEMENTED_REASON - assert "not implemented" in fp["reason"] + +def test_equal_size_edit_is_detected(): + """The regression this probe exists to catch: same length, changed content.""" + with requests_mock.Mocker() as m: + m.head(INSIDER_INTERFACES_URL, headers=_INTERFACES_HEADERS) + before = fetch_interfaces_fingerprint() + + with requests_mock.Mocker() as m: + m.head( + INSIDER_INTERFACES_URL, + headers={**_INTERFACES_HEADERS, "ETag": '"2f752a5-99999999999999"'}, + ) + after = fetch_interfaces_fingerprint() + + assert before["headers"] != after["headers"] + + +def test_schema_signal_is_populated_so_drift_is_not_auto_batched(): + """`drift_to_pr.classify_risk` tiers a diff as "routine" unless `headers` or + `checksums` moved. With both empty, a changed `track name=` format would be + swept into a batch telling the reviewer the schema signal was unchanged.""" + with requests_mock.Mocker() as m: + m.head(INSIDER_BED_URL, headers=_BED_HEADERS) + fp = fetch_fingerprint() + + from hvantk.core.plugin.api import PROBE_FINGERPRINT_IGNORED_KEYS + + schema_keys = {"headers", "checksums"} + assert not schema_keys & PROBE_FINGERPRINT_IGNORED_KEYS + assert fp["headers"], "headers must carry signal or every change reads as routine" + + +def test_missing_content_length_fails_closed(): + """Regression guard for the real failure hit while seeding the baseline: the + server gzips text/plain on the fly and then omits Content-Length entirely.""" + with requests_mock.Mocker() as m: + m.head(INSIDER_INTERFACES_URL, headers={"ETag": '"abc"'}) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_interfaces_fingerprint() + + +def test_empty_etag_fails_closed(): + """`ETag: ""` would normalize to '' and be stored as an empty digest, which + placeholder_baseline_reason then reads as a hand-seeded baseline -- trapping + the dataset in a probe_failed loop that regenerating cannot clear.""" + with requests_mock.Mocker() as m: + m.head( + INSIDER_INTERFACES_URL, + headers={"Content-Length": "49762981", "ETag": '""'}, + ) + with pytest.raises(DriftProbeError, match="no usable content signal"): + fetch_interfaces_fingerprint() + + +def test_compressed_response_fails_closed(): + """Identity is asserted request-side; a proxy may compress anyway, and the + recorded length would then describe the compressed body.""" + with requests_mock.Mocker() as m: + m.head( + INSIDER_INTERFACES_URL, + headers={ + "Content-Length": "123", + "ETag": '"abc"', + "Content-Encoding": "gzip", + }, + ) + with pytest.raises(DriftProbeError, match="gzip-encoded"): + fetch_interfaces_fingerprint() + + +@pytest.mark.parametrize( + ("probe", "url", "headers"), + [ + (fetch_fingerprint, INSIDER_BED_URL, _BED_HEADERS), + (fetch_interfaces_fingerprint, INSIDER_INTERFACES_URL, _INTERFACES_HEADERS), + ], +) +def test_probe_requests_identity_encoding(probe, url, headers): + """Without this the server compresses and Content-Length disappears.""" + with requests_mock.Mocker() as m: + m.head(url, headers=headers) + probe() + assert len(m.request_history) == 1 + assert m.request_history[0].headers.get("Accept-Encoding") == "identity" diff --git a/hvantk/skills/pqtl/SKILL.md b/hvantk/skills/pqtl/SKILL.md index f6b05ff3..57b17944 100644 --- a/hvantk/skills/pqtl/SKILL.md +++ b/hvantk/skills/pqtl/SKILL.md @@ -33,7 +33,13 @@ Fang allpairs inline via `hl.import_table`, parses GTEx variant IDs with `parse_gtex_variant_id` from `hvantk/core/utils/qtl_helpers.py`, derives SE as `|BETA / STAT|` (Fang files lack an SE column), and maps gene symbols to Ensembl gene IDs through a `GeneCatalogStreamer` (base class in -`hvantk/core/streamers/gene_catalog.py`). The drift probe is a stub. The +`hvantk/core/streamers/gene_catalog.py`). The drift probe checks the upstream +preprint's version metadata through medRxiv's public JSON API -- for a +publication-only source the publication *is* the upstream. + +**What it detects:** a new preprint version, or the preprint being published in a +journal. **What it cannot detect:** an in-place replacement of a supplementary file +under an unchanged version. The downloader is not implemented. ## Schema @@ -52,5 +58,6 @@ contains a registration-only test (`test_pqtl_metrics_registered`) and a skipped round-trip test (`test_pqtl_metrics_round_trip`). The `tests:` block in `plugin.yaml` declares plugin-relative fixture/snapshot paths (`tests/testdata/raw/pqtl`, `tests/snapshots/schema.json`, -`tests/snapshots/sample_rows.json`, `tests/drift_fingerprint.json`) that are -not yet populated. +`tests/snapshots/sample_rows.json`). Those remain unpopulated -- the statistics are +publication supplementary material with no redistributable fixture. The drift +fingerprint (`tests/drift_fingerprint.json`) IS populated, from a live probe run. diff --git a/hvantk/skills/pqtl/drift_probe.py b/hvantk/skills/pqtl/drift_probe.py index 11bd12c4..d294765b 100644 --- a/hvantk/skills/pqtl/drift_probe.py +++ b/hvantk/skills/pqtl/drift_probe.py @@ -1,20 +1,117 @@ -"""Drift probe for pqtl — documentation-only source (stub). +"""pqtl drift probe: preprint-version check against the medRxiv API. -pQTL summary statistics are publication-only (Fang et al. 2025 / GTEx); there is -no stable direct data URL to fingerprint. This probe returns a structured stub -sentinel so ``hvantk drift`` reports a visible WARNING (status="stub") rather -than a silent false-green. Replace with a real probe if a direct data URL becomes -available. See issue #177. +The Fang et al. pQTL summary statistics are published as supplementary material +to a preprint, which issue #177 recorded correctly: there is no direct data URL, +and acquisition stays manual. The conclusion that nothing could be probed does +not follow. For a publication-only source, the publication *is* the upstream, and +medRxiv exposes its metadata through a public JSON API. A new preprint version, +or the preprint being published in a journal, is precisely the event after which +the supplementary data may no longer match what an artifact was built from. + +Source: Fang et al. (2025), "Regulation of protein abundance in normal human +tissues", medRxiv, doi:10.1101/2025.01.10.25320181 -- 10,841 proteins across +>700 GTEx samples in five tissues, the cis-pQTL allpairs the builder consumes. + +Compared surface: the version number, the posting date, and the ``published`` +field (``"NA"`` until a journal version exists, then the journal DOI). The title +and abstract are deliberately excluded -- an editorial typo fix would otherwise +open a pull request carrying no information, the failure mode the hgnc probe was +corrected for. + +What this detects: a new preprint version, or journal publication. What it cannot +detect: an in-place replacement of a supplementary file under an unchanged +version. That limit is a property of how the source is published and is recorded +in SKILL.md so the coverage claim stays honest. """ + from __future__ import annotations -from hvantk.core.plugin.api import stub_fingerprint +from datetime import datetime, timezone + +import requests + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.core.utils.http import request_with_retry + +PROBE_VERSION = 2 + +PQTL_SOURCE_DOI = "10.1101/2025.01.10.25320181" +MEDRXIV_API_URL = f"https://api.biorxiv.org/details/medrxiv/{PQTL_SOURCE_DOI}" -_REASON = ( - "pQTL summary statistics are publication-only (Fang et al. 2025 / GTEx); " - "no stable direct data URL to fingerprint" -) +# Only these fields form the compared surface; see module docstring. `doi` is +# deliberately absent: the request URL is built FROM the DOI, so echoing it back +# is a constant that can never drift. +_COMPARED_FIELDS = ("version", "date", "published") + +_FILENAME = "medrxiv-preprint-metadata" +_TIMEOUT_S = (5.0, 15.0) def fetch_fingerprint() -> dict: - return stub_fingerprint(_REASON) + """Fingerprint the upstream preprint's version metadata.""" + try: + resp = request_with_retry( + "GET", MEDRXIV_API_URL, timeout=_TIMEOUT_S, allow_redirects=True + ) + resp.raise_for_status() + except requests.RequestException as exc: + raise DriftProbeError(f"HTTP failure: {exc}") from exc + + # Parsed in its own block: requests' JSONDecodeError subclasses both + # ValueError and RequestException, so decoding inside the block above would + # report a malformed body as an HTTP failure. + try: + payload = resp.json() + except ValueError as exc: + raise DriftProbeError(f"medRxiv API returned non-JSON: {exc}") from exc + + collection = payload.get("collection") or [] + # Fail closed. An empty collection means the DOI stopped resolving or the API + # changed shape, not that the preprint has no versions; recording it would + # bake an empty baseline that every later run compares equal to. + if not collection: + raise DriftProbeError( + f"medRxiv API returned no records for {PQTL_SOURCE_DOI}; the DOI or " + "the API shape has probably changed." + ) + + # Selected by version rather than by position. The API's ordering is not + # documented, and `collection[-1]` on a newest-first response would pin the + # OLDEST record -- so a v2 posting, the single event this probe exists to + # detect, would compare equal to the baseline and report clean forever. + def _version_of(record: dict) -> int: + raw = record.get("version") + try: + return int(raw) + except (TypeError, ValueError): + return -1 + + latest = max(collection, key=_version_of) + version = _version_of(latest) + # Fail closed rather than stringifying a missing field. `str(None)` yields the + # literal "None", which is truthy and indistinguishable from a real version + # label, and the bot would commit it as the baseline. + if version < 0: + raise DriftProbeError( + f"medRxiv record for {PQTL_SOURCE_DOI} carried no usable version " + f"field (got {latest.get('version')!r}); the API shape has probably " + "changed." + ) + + compared = {field: latest.get(field) for field in _COMPARED_FIELDS} + # Normalised so an API that switches "1" to 1 does not read as drift. + compared["version"] = str(version) + + return { + "probe_version": PROBE_VERSION, + "source_version": str(version), + # The metadata IS the signal; a sha256 over it would be a pure function of + # values already in the compared surface. + "headers": {_FILENAME: compared}, + "checksums": {}, + "informational": { + "title": latest.get("title"), + "versions_listed": len(collection), + }, + "fetched_at": datetime.now(timezone.utc).isoformat(), + } diff --git a/hvantk/skills/pqtl/tests/drift_fingerprint.json b/hvantk/skills/pqtl/tests/drift_fingerprint.json new file mode 100644 index 00000000..d65decc3 --- /dev/null +++ b/hvantk/skills/pqtl/tests/drift_fingerprint.json @@ -0,0 +1,17 @@ +{ + "probe_version": 2, + "source_version": "1", + "headers": { + "medrxiv-preprint-metadata": { + "version": "1", + "date": "2025-01-13", + "published": "NA" + } + }, + "checksums": {}, + "informational": { + "title": "Regulation of protein abundance in normal human tissues", + "versions_listed": 1 + }, + "fetched_at": "2026-09-01T07:32:19.306546+00:00" +} diff --git a/hvantk/skills/pqtl/tests/test_drift_probe.py b/hvantk/skills/pqtl/tests/test_drift_probe.py new file mode 100644 index 00000000..9fe715dd --- /dev/null +++ b/hvantk/skills/pqtl/tests/test_drift_probe.py @@ -0,0 +1,127 @@ +"""pqtl drift probe should fingerprint the upstream preprint's version metadata. + +Runs OFFLINE via requests_mock. The pQTL statistics ship as supplementary +material to a preprint, so the publication is the upstream (issue #177 recorded +the absence of a direct data URL correctly). +""" + +from __future__ import annotations + +import pytest +import requests_mock + +from hvantk.core.plugin.api import DriftProbeError +from hvantk.skills.pqtl.drift_probe import ( + MEDRXIV_API_URL, + PQTL_SOURCE_DOI, + fetch_fingerprint, +) + + +def _payload(**overrides): + record = { + "title": "Regulation of protein abundance in normal human tissues", + "doi": PQTL_SOURCE_DOI, + "date": "2025-01-13", + "version": "1", + "published": "NA", + } + record.update(overrides) + return {"messages": [{"status": "ok"}], "collection": [record]} + + +def test_fetch_fingerprint_shape(): + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload()) + fp = fetch_fingerprint() + + assert fp["source_version"] == "1" + assert fp["headers"]["medrxiv-preprint-metadata"]["published"] == "NA" + + +def test_journal_publication_moves_the_checksum(): + """`published` flipping from NA to a journal DOI is the event that most likely + means the supplementary data no longer matches what an artifact was built from.""" + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload()) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload(published="10.1038/s41588-025-0000-0")) + after = fetch_fingerprint() + + assert before["headers"] != after["headers"] + + +def test_editorial_title_change_does_not_move_the_checksum(): + """Title and abstract are outside the compared surface, so a typo fix must not + open a pull request carrying no information.""" + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload()) + before = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload(title="Regulation of protein abundance")) + after = fetch_fingerprint() + + assert before["headers"] == after["headers"] + + +@pytest.mark.parametrize("newest_first", [False, True]) +def test_newest_version_wins_regardless_of_response_order(newest_first): + """The API's ordering is not documented. `collection[-1]` on a newest-first + response pinned the OLDEST record, so a v2 posting -- the single event this + probe exists to detect -- compared equal to the baseline forever.""" + records = [ + {"version": "1", "date": "2025-01-13", "published": "NA", + "doi": PQTL_SOURCE_DOI, "title": "t"}, + {"version": "2", "date": "2025-06-01", "published": "NA", + "doi": PQTL_SOURCE_DOI, "title": "t"}, + ] + if newest_first: + records.reverse() + + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json={"collection": records}) + fp = fetch_fingerprint() + + assert fp["source_version"] == "2" + assert fp["informational"]["versions_listed"] == 2 + + +def test_missing_version_fails_closed(): + """`str(None)` yields the literal "None", which is truthy and indistinguishable + from a real version label; the bot would commit it as the baseline.""" + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload(version=None)) + with pytest.raises(DriftProbeError, match="no usable version"): + fetch_fingerprint() + + +def test_integer_version_does_not_read_as_drift(): + """An API that switches "1" to 1 must not flip the compared surface.""" + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload()) + as_text = fetch_fingerprint() + + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload(version=1)) + as_int = fetch_fingerprint() + + assert as_text["headers"] == as_int["headers"] + + +def test_doi_is_not_in_the_compared_surface(): + """The request URL is built FROM the DOI, so echoing it back is a constant.""" + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json=_payload()) + fp = fetch_fingerprint() + + assert "doi" not in fp["headers"]["medrxiv-preprint-metadata"] + + +def test_empty_collection_fails_closed(): + with requests_mock.Mocker() as m: + m.get(MEDRXIV_API_URL, json={"collection": []}) + with pytest.raises(DriftProbeError, match="no records"): + fetch_fingerprint() diff --git a/hvantk/tests/test_committed_drift_baselines.py b/hvantk/tests/test_committed_drift_baselines.py new file mode 100644 index 00000000..571e19a2 --- /dev/null +++ b/hvantk/tests/test_committed_drift_baselines.py @@ -0,0 +1,100 @@ +"""Every committed drift baseline must be real probe output. + +`test_plugin_contract_artifacts` checks only that the file EXISTS. That cannot +support the claim the ledger leans on -- that a baseline "was captured from a live +probe run" -- and it is exactly the gap `placeholder_baseline_reason` was written +to close after six hand-seeded baselines produced a permanent, meaningless +`drifted` verdict while looking maximally alive. + +A hand-edit (a merge-conflict resolution, a redaction, a copy-paste of another +plugin's shape) re-creates those dead comparators, and the existence ratchet would +still report the dataset complete. These tests read the committed JSON directly, so +they need no network and run in the default selection. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from hvantk.core.plugin.api import ( + PROBE_STATUS_STUB, + placeholder_baseline_reason, +) + +SKILLS_DIR = Path(__file__).resolve().parents[1] / "skills" + + +def _committed_baselines() -> dict[str, Path]: + """Map ``provider:dataset`` -> the drift baseline its manifest declares.""" + found: dict[str, Path] = {} + for manifest_path in sorted(SKILLS_DIR.glob("*/plugin.yaml")): + manifest = yaml.safe_load(manifest_path.read_text()) + provider = manifest.get("name") or manifest_path.parent.name + for dataset in manifest.get("datasets", []): + rel = (dataset.get("tests") or {}).get("drift_fingerprint") + if not rel: + continue + path = manifest_path.parent / rel + if path.exists(): + found[f"{provider}:{dataset['name']}"] = path + return found + + +BASELINES = _committed_baselines() +assert BASELINES, "no committed drift baselines discovered" + + +@pytest.mark.parametrize("name", sorted(BASELINES)) +def test_baseline_is_valid_json_object(name): + payload = json.loads(BASELINES[name].read_text()) + assert isinstance(payload, dict), f"{name}: baseline is not a JSON object" + + +@pytest.mark.parametrize("name", sorted(BASELINES)) +def test_baseline_is_not_hand_seeded(name): + """The check drift_runner performs before diffing, applied at rest. + + Without this a placeholder or empty-digest baseline can be committed and only + surfaces later as a permanent probe_failed that regenerating cannot clear. + """ + payload = json.loads(BASELINES[name].read_text()) + reason = placeholder_baseline_reason(payload) + assert reason is None, f"{name}: {reason}" + + +@pytest.mark.parametrize("name", sorted(BASELINES)) +def test_baseline_records_a_real_fetch(name): + """A captured baseline carries the timestamp of the run that produced it.""" + payload = json.loads(BASELINES[name].read_text()) + if payload.get("probe_status") == PROBE_STATUS_STUB: + pytest.skip("stub sentinel: no fetch to record") + fetched_at = payload.get("fetched_at") + assert isinstance(fetched_at, str) and fetched_at, f"{name}: no fetched_at" + assert not fetched_at.startswith("1970-01-01"), f"{name}: epoch fetched_at" + + +@pytest.mark.parametrize("name", sorted(BASELINES)) +def test_baseline_carries_a_comparable_signal(name): + """A baseline whose entire compared surface is empty compares equal forever. + + `placeholder_baseline_reason` deliberately does not flag an empty `checksums` + map (peptideatlas legitimately ships one), so nothing else catches the case + where `headers`, `checksums` AND `source_version` are all empty at once. + """ + payload = json.loads(BASELINES[name].read_text()) + if payload.get("probe_status") == PROBE_STATUS_STUB: + pytest.skip("stub sentinel: no comparable signal by design") + signal = ( + payload.get("headers") + or payload.get("checksums") + or payload.get("extras") + or payload.get("source_version") + ) + assert signal, ( + f"{name}: baseline carries no compared signal at all; every future run " + "would report clean regardless of what upstream does" + ) diff --git a/hvantk/tests/test_docs_claims.py b/hvantk/tests/test_docs_claims.py index b6764bef..c0496c01 100644 --- a/hvantk/tests/test_docs_claims.py +++ b/hvantk/tests/test_docs_claims.py @@ -19,7 +19,9 @@ `core/utils/writers.py`, `algorithms/training_sets/`, `tools/build/` -- while the parallel tree in README.md was corrected by hand. A contributor following the stale one is told to put new code in directories that are not - there, in a layer that would violate the dependency rule. + there, in a layer that would violate the dependency rule. That divergence is + now structurally impossible: README.md's duplicate tree was replaced by a + link, so architecture.md holds the only one and these tests guard it there. Deliberately non-Hail so it runs in the default suite. It does import every subcommand module (resolving a documented command imports the module that @@ -197,7 +199,7 @@ def _is_illustrative(path: str) -> bool: return "*" in path or "<" in path or path.endswith("...") -@pytest.mark.parametrize("doc", ["README.md", "docs_site/architecture.md"]) +@pytest.mark.parametrize("doc", ["docs_site/architecture.md"]) def test_structure_tree_names_only_real_paths(doc: str): """Every entry drawn in a project-structure tree must exist at that path. @@ -223,7 +225,7 @@ def test_structure_tree_names_only_real_paths(doc: str): def test_the_tree_parser_reconstructs_nested_paths(): """Guard the guard: if depth parsing broke, everything above goes vacuous.""" - drawn = _tree_paths((REPO_ROOT / "README.md").read_text()) + drawn = _tree_paths((REPO_ROOT / "docs_site" / "architecture.md").read_text()) assert "core/models" in drawn, sorted(d for d in drawn if "models" in d) assert "core" in drawn and len(drawn) > 20 diff --git a/hvantk/tests/test_plugin_contract_artifacts.py b/hvantk/tests/test_plugin_contract_artifacts.py index ca6212e0..63fd1a7a 100644 --- a/hvantk/tests/test_plugin_contract_artifacts.py +++ b/hvantk/tests/test_plugin_contract_artifacts.py @@ -42,17 +42,26 @@ # 3. simply never seeded, though the builder runs from a committed fixture -- the # majority, and the ones the follow-up work removes from this list. # -# clingen, gencc and hgnc were removed from this list once their snapshots landed, and -# dbnsfp / gnomad-metrics now lack only a drift fingerprint. The -# remaining fingerprint gaps are a separate concern from snapshots: a probe has to be run -# against the live upstream, which the snapshot tests deliberately never touch. +# clingen, gencc and hgnc were removed from this list once their snapshots landed. # (gevir shipped its drift fingerprint as part of the gevir plugin-review work, so it # left this list.) # -# (dbnsfp / gnomad-metrics previously appeared here for a different -# reason -- they declared a plugin-local fixture dir that was never created while the -# tests read one under hvantk/tests/testdata/raw/. Their manifests now point at the real -# shared location, so only their snapshot/fingerprint files remain outstanding.) +# dbnsfp and gnomad-metrics left it too, once real drift probes replaced their stub +# sentinels and their baselines were captured from live probe runs. Both had been +# recorded under issue #177 as having no probeable URL; re-checking showed the gnomAD +# constraint tables sit in a public GCS bucket that returns an MD5 ETag, and that the +# dbNSFP landing page -- though its advertised S3 archives are all dead (issue #321) -- +# still exposes a stable release list. +# +# The five entries below are NOT the datasets without a drift probe: every dataset in +# the tree now ships a live one. They are the datasets still missing a *fixture* or +# *snapshot*, for the reasons in the three causes above -- alphagenome, cosmic-cgc and +# pqtl have no committable static artifact or no redistributable rows, while +# expression-atlas and peptideatlas await fixtures derived by truncation. +# +# (dbnsfp / gnomad-metrics previously appeared here twice over: first for a fixture dir +# that was never created, then for a missing drift fingerprint. Both are resolved and +# neither is listed any more.) # # uniprot-ptm and both cptac datasets had no committed fixture at all -- their round-trip # tests synthesized inputs into tmp_path. Small fixtures were committed for each, so all @@ -63,13 +72,11 @@ # large to use directly (116k transcripts x 320 samples), so a fixture must be *derived* # by truncation rather than copied. KNOWN_INCOMPLETE: dict[str, tuple[str, ...]] = { - "alphagenome:predictions": ARTIFACT_FIELDS, - "cosmic-cgc:submissions": ARTIFACT_FIELDS, - "dbnsfp:variants": ("drift_fingerprint",), + "alphagenome:predictions": ("fixture", "schema_snapshot", "row_snapshot"), + "cosmic-cgc:submissions": ("fixture", "schema_snapshot", "row_snapshot"), "expression-atlas:dataset": ("schema_snapshot", "row_snapshot"), - "gnomad-metrics:metrics": ("drift_fingerprint",), "peptideatlas:phospho": ("schema_snapshot", "row_snapshot"), - "pqtl:metrics": ARTIFACT_FIELDS, + "pqtl:metrics": ("fixture", "schema_snapshot", "row_snapshot"), } diff --git a/hvantk/tests/test_probe_normalize_etag.py b/hvantk/tests/test_probe_normalize_etag.py new file mode 100644 index 00000000..6813d251 --- /dev/null +++ b/hvantk/tests/test_probe_normalize_etag.py @@ -0,0 +1,42 @@ +"""`normalize_etag` must reduce every real ETag form to one bare tag. + +The shipped-once bug was `str.strip('"')`, which strips a character SET from both +ends: `W/"abc"` kept its `W/` prefix because the leading `W` blocked the left +strip. Because the drift bot regenerates drifted baselines automatically, one +mangled value bakes in permanently. +""" + +from __future__ import annotations + +import pytest + +from hvantk.core.plugin.api import normalize_etag + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ('"abc123"', "abc123"), # strong + ('W/"abc123"', "abc123"), # weak -- the strip() bug + ('w/"abc123"', "abc123"), # weak, lowercase + ('"abc123"-gzip', "abc123"), # transform suffix appended by mod_deflate + (' "abc123" ', "abc123"), # surrounding whitespace + ("abc123", "abc123"), # unquoted, seen from some proxies + ], +) +def test_every_form_reduces_to_the_same_tag(raw, expected): + assert normalize_etag(raw) == expected + + +@pytest.mark.parametrize("raw", [None, "", '""', " ", 'W/""']) +def test_absent_or_empty_becomes_none(raw): + """Callers fail closed on None. Returning '' instead would be recorded as an + empty digest, which placeholder_baseline_reason later reads as a hand-seeded + baseline -- an unbreakable probe_failed loop.""" + assert normalize_etag(raw) is None + + +def test_the_old_strip_bug_would_fail_these(): + """Documents precisely what regressed, so a revert to strip('"') is caught.""" + assert 'W/"abc"'.strip('"') == 'W/"abc' # the bug + assert normalize_etag('W/"abc"') == "abc" # the fix diff --git a/hvantk/tests/test_pyproject_extras.py b/hvantk/tests/test_pyproject_extras.py index 6259f638..6b5595d2 100644 --- a/hvantk/tests/test_pyproject_extras.py +++ b/hvantk/tests/test_pyproject_extras.py @@ -4,15 +4,19 @@ pulled by an extras install, breaking `import cptac`. The ptm extra must declare `sorted-nearest` explicitly. -The docs half exists because the extras table is duplicated in THREE places -- the -`[project.optional-dependencies]` table, README.md, and -docs_site/getting-started/installation.md -- -and only the first is executable. Three separate hand-fixes to the two prose copies were -needed in as many sessions, two of them caught only by adversarial review, and a fourth -drift (psroc/ancestry/ml missing scipy, ptm missing sorted-nearest -- eight wrong cells) -survived a release. A reader following a wrong table installs an environment that cannot run -the command the table promises, which is exactly the failure `pip install hvantk[constraint]` -produced. Deliberately non-Hail so it runs in the default suite. +The docs half exists because the extras table is duplicated -- the +`[project.optional-dependencies]` table and +docs_site/getting-started/installation.md -- and only the first is executable. Three +separate hand-fixes to the prose copies were needed in as many sessions, two of them +caught only by adversarial review, and a fourth drift (psroc/ancestry/ml missing scipy, +ptm missing sorted-nearest -- eight wrong cells) survived a release. A reader following a +wrong table installs an environment that cannot run the command the table promises, which +is exactly the failure `pip install hvantk[constraint]` produced. + +There were THREE copies until README.md's was replaced by a pointer to installation.md; +that removal is why one prose table is now guarded rather than two. Keep it that way: a +new copy of this table anywhere is a new thing to drift, and it belongs in this list if it +is added. Deliberately non-Hail so it runs in the default suite. """ from __future__ import annotations @@ -22,11 +26,11 @@ import pytest ROOT = Path(__file__).resolve().parents[2] -# (path, the column holding the dependency list). Both tables are markdown pipe tables whose -# first column is the extra name in backticks; they differ in column order, so each doc names -# its own header rather than assuming a position. +# (path, the column holding the dependency list). The table is a markdown pipe table whose +# first column is the extra name in backticks. Each doc names its own header rather than +# assuming a position, so a second copy with a different column order can be added here +# without touching the parser. DOC_TABLES = [ - (ROOT / "README.md", "Pulls in"), (ROOT / "docs_site" / "getting-started" / "installation.md", "Pulls in"), ] diff --git a/hvantk/tests/test_run_builder_probe_degradation.py b/hvantk/tests/test_run_builder_probe_degradation.py new file mode 100644 index 00000000..90becde4 --- /dev/null +++ b/hvantk/tests/test_run_builder_probe_degradation.py @@ -0,0 +1,117 @@ +"""A build must survive an unreachable drift probe. + +The probe supplies provenance metadata, not build input. Every +manual-acquisition plugin is built from a hand-staged file, often on a node with +no egress; before the documentation-only plugins gained live probes their probes +were pure in-process calls, so those builds needed no network at all. Letting a +DriftProbeError propagate would make `hvantk reprocess --skip-download +--no-check-drift` fail offline -- and `--no-check-drift` gates only the +post-build check, not the probe call in run_builder. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from hvantk.core.plugin.api import DriftProbeError, PROBE_UNAVAILABLE_TOKEN +from hvantk.core.plugin.run_builder import _coerce_fingerprint, run_builder_for_spec + + +class _Artifact: + """Minimal stand-in for a typed artifact: saves, and carries provenance.""" + + def __init__(self): + self.saved_to = None + self.provenance = SimpleNamespace(schema_id=None) + + def save(self, path): + self.saved_to = path + + +def _spec(probe): + spec = Mock() + spec.name = "insider:variants" + spec.artifact_type = _Artifact + spec.drift_probe = probe + spec.builder = lambda parsed_input, ctx, **params: _Artifact() + # Mock auto-creates truthy attributes; the schema_id check would then + # dereference .provenance on the stand-in artifact. + spec.schema_id = None + return spec + + +def test_unreachable_probe_does_not_abort_the_build(tmp_path, caplog): + spec = _spec(Mock(side_effect=DriftProbeError("HTTP failure: no route to host"))) + + provenance = run_builder_for_spec( + spec, + parsed_input="in", + output_path=str(tmp_path / "out.ht"), + plugin_version="0.1.0", + ) + + assert provenance is not None + assert "drift probe could not reach its source" in caplog.text + + +def test_provenance_never_implies_a_probe_ran(tmp_path): + """The fallback is self-describing, not a synthesized digest -- the same + reasoning as STUB_FINGERPRINT_TOKEN.""" + captured = {} + + def builder(parsed_input, ctx, **params): + captured["fingerprint"] = ctx.source_fingerprint + return _Artifact() + + spec = _spec(Mock(side_effect=DriftProbeError("unreachable"))) + spec.builder = builder + + run_builder_for_spec( + spec, + parsed_input="in", + output_path=str(tmp_path / "o.ht"), + plugin_version="0.1.0", + ) + + assert captured["fingerprint"] == PROBE_UNAVAILABLE_TOKEN + assert not captured["fingerprint"].startswith("sha256:") + + +def test_a_working_probe_still_stamps_a_real_fingerprint(tmp_path): + """The degradation must not mask a healthy probe.""" + captured = {} + + def builder(parsed_input, ctx, **params): + captured["fingerprint"] = ctx.source_fingerprint + return _Artifact() + + probe_result = {"probe_version": 2, "headers": {"f": {"content_length": "1"}}} + spec = _spec(Mock(return_value=probe_result)) + spec.builder = builder + + run_builder_for_spec( + spec, + parsed_input="in", + output_path=str(tmp_path / "o.ht"), + plugin_version="0.1.0", + ) + + assert captured["fingerprint"] == _coerce_fingerprint(probe_result, "x") + assert captured["fingerprint"] != PROBE_UNAVAILABLE_TOKEN + + +def test_a_contract_violation_still_raises(tmp_path): + """Only DriftProbeError degrades; a probe returning a non-dict is a bug and + must stay loud.""" + spec = _spec(Mock(return_value="not-a-dict")) + + with pytest.raises(Exception): + run_builder_for_spec( + spec, + parsed_input="in", + output_path=str(tmp_path / "o.ht"), + plugin_version="0.1.0", + )